changeset 1405:9a29868e2610

Better error handling for missing jvmstat counters When running Thermostat with OpenJDK 8, jvmstat falsely reports the number of GC generations. This causes us to query performance counters for a non-existent generation. This patch adds error handling so the agent doesn't crash with a NullPointerException on these non-existent counters. The patch also includes test cases for these error conditions. Reviewed-by: omajid Review-thread: http://icedtea.classpath.org/pipermail/thermostat/2014-March/009355.html PR1800
author Elliott Baron <ebaron@redhat.com>
date Mon, 10 Mar 2014 18:35:14 -0400
parents 72ad85995de7
children 60643e432434
files vm-classstat/agent/src/main/java/com/redhat/thermostat/vm/classstat/agent/internal/VmClassStatDataExtractor.java vm-classstat/agent/src/main/java/com/redhat/thermostat/vm/classstat/agent/internal/VmClassStatVmListener.java vm-classstat/agent/src/test/java/com/redhat/thermostat/vm/classstat/agent/internal/VmClassStatVmListenerTest.java vm-gc/agent/src/main/java/com/redhat/thermostat/vm/gc/agent/internal/VmGcDataExtractor.java vm-gc/agent/src/main/java/com/redhat/thermostat/vm/gc/agent/internal/VmGcVmListener.java vm-gc/agent/src/test/java/com/redhat/thermostat/vm/gc/agent/internal/VmGcVmListenerTest.java vm-memory/agent/src/main/java/com/redhat/thermostat/vm/memory/agent/internal/VmMemoryDataExtractor.java vm-memory/agent/src/main/java/com/redhat/thermostat/vm/memory/agent/internal/VmMemoryVmListener.java vm-memory/agent/src/test/java/com/redhat/thermostat/vm/memory/agent/internal/VmMemoryVmListenerTest.java
diffstat 9 files changed, 534 insertions(+), 48 deletions(-) [+]
line wrap: on
line diff
--- a/vm-classstat/agent/src/main/java/com/redhat/thermostat/vm/classstat/agent/internal/VmClassStatDataExtractor.java	Thu May 22 10:49:54 2014 +0200
+++ b/vm-classstat/agent/src/main/java/com/redhat/thermostat/vm/classstat/agent/internal/VmClassStatDataExtractor.java	Mon Mar 10 18:35:14 2014 -0400
@@ -65,7 +65,7 @@
         this.update = update;
     }
 
-    public long getLoadedClasses() throws VmUpdateException {
+    public Long getLoadedClasses() throws VmUpdateException {
         return update.getPerformanceCounterLong("java.cls.loadedClasses");
     }
 
--- a/vm-classstat/agent/src/main/java/com/redhat/thermostat/vm/classstat/agent/internal/VmClassStatVmListener.java	Thu May 22 10:49:54 2014 +0200
+++ b/vm-classstat/agent/src/main/java/com/redhat/thermostat/vm/classstat/agent/internal/VmClassStatVmListener.java	Mon Mar 10 18:35:14 2014 -0400
@@ -53,6 +53,8 @@
     private final VmClassStatDAO dao;
     private final String vmId;
     private final String writerId;
+    
+    private boolean error;
 
     VmClassStatVmListener(String writerId, VmClassStatDAO dao, String vmId) {
         this.dao = dao;
@@ -64,12 +66,26 @@
     public void countersUpdated(VmUpdate update) {
         VmClassStatDataExtractor extractor = new VmClassStatDataExtractor(update);
         try {
-            long loadedClasses = extractor.getLoadedClasses();
-            long timestamp = System.currentTimeMillis();
-            VmClassStat stat = new VmClassStat(writerId, vmId, timestamp, loadedClasses);
-            dao.putVmClassStat(stat);
+            Long loadedClasses = extractor.getLoadedClasses();
+            if (loadedClasses != null) {
+                long timestamp = System.currentTimeMillis();
+                VmClassStat stat = new VmClassStat(writerId, vmId, timestamp, loadedClasses);
+                dao.putVmClassStat(stat);
+            }
+            else {
+                logWarningOnce("Unable to determine number of loaded classes for VM " 
+                        + vmId);
+            }
         } catch (VmUpdateException e) {
-            logger.log(Level.WARNING, "error gathering class info for vm " + vmId, e);
+            logger.log(Level.WARNING, "Error gathering class info for VM " + vmId, e);
+        }
+    }
+    
+    private void logWarningOnce(String message) {
+        if (!error) {
+            logger.log(Level.WARNING, message);
+            logger.log(Level.WARNING, "Further warnings will be ignored");
+            error = true;
         }
     }
 
--- a/vm-classstat/agent/src/test/java/com/redhat/thermostat/vm/classstat/agent/internal/VmClassStatVmListenerTest.java	Thu May 22 10:49:54 2014 +0200
+++ b/vm-classstat/agent/src/test/java/com/redhat/thermostat/vm/classstat/agent/internal/VmClassStatVmListenerTest.java	Mon Mar 10 18:35:14 2014 -0400
@@ -37,9 +37,11 @@
 package com.redhat.thermostat.vm.classstat.agent.internal;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
@@ -101,5 +103,15 @@
 
         verifyNoMoreInteractions(dao);
     }
+    
+    @Test
+    public void testMonitorUpdatedClassStatNoCounter() throws Exception {
+        VmUpdate update = mock(VmUpdate.class);
+        when(update.getPerformanceCounterLong(eq("java.cls.loadedClasses"))).thenReturn(null);
+
+        listener.countersUpdated(update);
+
+        verify(dao, never()).putVmClassStat(any(VmClassStat.class));
+    }
 }
 
--- a/vm-gc/agent/src/main/java/com/redhat/thermostat/vm/gc/agent/internal/VmGcDataExtractor.java	Thu May 22 10:49:54 2014 +0200
+++ b/vm-gc/agent/src/main/java/com/redhat/thermostat/vm/gc/agent/internal/VmGcDataExtractor.java	Mon Mar 10 18:35:14 2014 -0400
@@ -65,7 +65,7 @@
         this.update = update;
     }
 
-    public long getTotalCollectors() throws VmUpdateException {
+    public Long getTotalCollectors() throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.policy.collectors");
     }
 
@@ -73,11 +73,11 @@
         return update.getPerformanceCounterString("sun.gc.collector." + collector + ".name");
     }
 
-    public long getCollectorTime(long collector) throws VmUpdateException {
+    public Long getCollectorTime(long collector) throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.collector." + collector + ".time");
     }
 
-    public long getCollectorInvocations(long collector) throws VmUpdateException {
+    public Long getCollectorInvocations(long collector) throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.collector." + collector + ".invocations");
     }
 
--- a/vm-gc/agent/src/main/java/com/redhat/thermostat/vm/gc/agent/internal/VmGcVmListener.java	Thu May 22 10:49:54 2014 +0200
+++ b/vm-gc/agent/src/main/java/com/redhat/thermostat/vm/gc/agent/internal/VmGcVmListener.java	Mon Mar 10 18:35:14 2014 -0400
@@ -53,6 +53,8 @@
     private final String vmId;
     private final VmGcStatDAO gcDAO;
     private final String writerId;
+    
+    private boolean error;
 
     public VmGcVmListener(String writerId, VmGcStatDAO vmGcStatDao, String vmId) {
         gcDAO = vmGcStatDao;
@@ -68,17 +70,48 @@
 
     void recordGcStat(VmGcDataExtractor extractor) {
         try {
-            long collectors = extractor.getTotalCollectors();
-            for (int i = 0; i < collectors; i++) {
-                long timestamp = System.currentTimeMillis();
-                VmGcStat stat = new VmGcStat(writerId, vmId, timestamp,
-                        extractor.getCollectorName(i),
-                        extractor.getCollectorInvocations(i),
-                        extractor.getCollectorTime(i));
-                gcDAO.putVmGcStat(stat);
+            Long collectors = extractor.getTotalCollectors();
+            if (collectors != null) {
+                for (int i = 0; i < collectors; i++) {
+                    long timestamp = System.currentTimeMillis();
+                    String name = extractor.getCollectorName(i);
+                    if (name != null) {
+                        Long invocations = extractor.getCollectorInvocations(i);
+                        if (invocations != null) {
+                            Long time = extractor.getCollectorTime(i);
+                            if (time != null) {
+                                VmGcStat stat = new VmGcStat(writerId, vmId, timestamp,
+                                        name, invocations, time);
+                                gcDAO.putVmGcStat(stat);
+                            }
+                            else {
+                                logWarningOnce("Unable to determine time spent by collector " 
+                                        + i + " for VM " + vmId);
+                            }
+                        }
+                        else {
+                            logWarningOnce("Unable to determine number of invocations of collector " 
+                                    + i + " for VM " + vmId);
+                        }
+                    }
+                    else {
+                        logWarningOnce("Unable to determine name of collector " + i + " for VM " + vmId);
+                    }
+                }
+            }
+            else {
+                logWarningOnce("Unable to determine number of collectors for VM " + vmId);
             }
         } catch (VmUpdateException e) {
-            logger.log(Level.WARNING, "error gathering gc info for vm " + vmId, e);
+            logger.log(Level.WARNING, "Error gathering GC info for VM " + vmId, e);
+        }
+    }
+    
+    private void logWarningOnce(String message) {
+        if (!error) {
+            logger.log(Level.WARNING, message);
+            logger.log(Level.WARNING, "Further warnings will be ignored");
+            error = true;
         }
     }
 
--- a/vm-gc/agent/src/test/java/com/redhat/thermostat/vm/gc/agent/internal/VmGcVmListenerTest.java	Thu May 22 10:49:54 2014 +0200
+++ b/vm-gc/agent/src/test/java/com/redhat/thermostat/vm/gc/agent/internal/VmGcVmListenerTest.java	Mon Mar 10 18:35:14 2014 -0400
@@ -37,7 +37,9 @@
 package com.redhat.thermostat.vm.gc.agent.internal;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -105,5 +107,63 @@
             assertEquals(GC_TIMES[i], (Long) stat.getWallTime());
         }
     }
+    
+    @Test
+    public void testRecordMemoryStatNoName() throws VmUpdateException {
+        when(extractor.getCollectorName(1)).thenReturn(null);
+        vmListener.recordGcStat(extractor);
+        ArgumentCaptor<VmGcStat> captor = ArgumentCaptor.forClass(VmGcStat.class);
+        verify(vmGcStatDAO).putVmGcStat(captor.capture());
+        List<VmGcStat> gcStats = captor.getAllValues();
+        
+        // Verify second collector skipped
+        assertEquals(1, gcStats.size());
+        
+        VmGcStat stat = gcStats.get(0);
+        assertEquals(GC_NAMES[0], stat.getCollectorName());
+        assertEquals(GC_INVOCS[0], (Long) stat.getRunCount());
+        assertEquals(GC_TIMES[0], (Long) stat.getWallTime());
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoInvocations() throws VmUpdateException {
+        when(extractor.getCollectorInvocations(1)).thenReturn(null);
+        vmListener.recordGcStat(extractor);
+        ArgumentCaptor<VmGcStat> captor = ArgumentCaptor.forClass(VmGcStat.class);
+        verify(vmGcStatDAO).putVmGcStat(captor.capture());
+        List<VmGcStat> gcStats = captor.getAllValues();
+        
+        // Verify second collector skipped
+        assertEquals(1, gcStats.size());
+        
+        VmGcStat stat = gcStats.get(0);
+        assertEquals(GC_NAMES[0], stat.getCollectorName());
+        assertEquals(GC_INVOCS[0], (Long) stat.getRunCount());
+        assertEquals(GC_TIMES[0], (Long) stat.getWallTime());
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoTime() throws VmUpdateException {
+        when(extractor.getCollectorTime(1)).thenReturn(null);
+        vmListener.recordGcStat(extractor);
+        ArgumentCaptor<VmGcStat> captor = ArgumentCaptor.forClass(VmGcStat.class);
+        verify(vmGcStatDAO).putVmGcStat(captor.capture());
+        List<VmGcStat> gcStats = captor.getAllValues();
+        
+        // Verify second collector skipped
+        assertEquals(1, gcStats.size());
+        
+        VmGcStat stat = gcStats.get(0);
+        assertEquals(GC_NAMES[0], stat.getCollectorName());
+        assertEquals(GC_INVOCS[0], (Long) stat.getRunCount());
+        assertEquals(GC_TIMES[0], (Long) stat.getWallTime());
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoTotal() throws VmUpdateException {
+        when(extractor.getTotalCollectors()).thenReturn(null);
+        vmListener.recordGcStat(extractor);
+        verify(vmGcStatDAO, never()).putVmGcStat(any(VmGcStat.class));
+    }
 }
 
--- a/vm-memory/agent/src/main/java/com/redhat/thermostat/vm/memory/agent/internal/VmMemoryDataExtractor.java	Thu May 22 10:49:54 2014 +0200
+++ b/vm-memory/agent/src/main/java/com/redhat/thermostat/vm/memory/agent/internal/VmMemoryDataExtractor.java	Mon Mar 10 18:35:14 2014 -0400
@@ -66,7 +66,7 @@
         this.update = update;
     }
     
-    public long getTotalGcGenerations() throws VmUpdateException {
+    public Long getTotalGcGenerations() throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.policy.generations");
     }
 
@@ -74,11 +74,11 @@
         return update.getPerformanceCounterString("sun.gc.generation." + generation + ".name");
     }
 
-    public long getGenerationCapacity(long generation) throws VmUpdateException {
+    public Long getGenerationCapacity(long generation) throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.generation." + generation + ".capacity");
     }
 
-    public long getGenerationMaxCapacity(long generation) throws VmUpdateException {
+    public Long getGenerationMaxCapacity(long generation) throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.generation." + generation + ".maxCapacity");
     }
 
@@ -92,7 +92,7 @@
         return collector;
     }
 
-    public long getTotalSpaces(long generation) throws VmUpdateException {
+    public Long getTotalSpaces(long generation) throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.generation." + generation + ".spaces");
     }
 
@@ -100,15 +100,15 @@
         return update.getPerformanceCounterString("sun.gc.generation." + generation + ".space." + space + ".name");
     }
 
-    public long getSpaceCapacity(long generation, long space) throws VmUpdateException {
+    public Long getSpaceCapacity(long generation, long space) throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.generation." + generation + ".space." + space + ".capacity");
     }
 
-    public long getSpaceMaxCapacity(long generation, long space) throws VmUpdateException {
+    public Long getSpaceMaxCapacity(long generation, long space) throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.generation." + generation + ".space." + space + ".maxCapacity");
     }
 
-    public long getSpaceUsed(long generation, long space) throws VmUpdateException {
+    public Long getSpaceUsed(long generation, long space) throws VmUpdateException {
         return update.getPerformanceCounterLong("sun.gc.generation." + generation + ".space." + space + ".used");
     }
 
--- a/vm-memory/agent/src/main/java/com/redhat/thermostat/vm/memory/agent/internal/VmMemoryVmListener.java	Thu May 22 10:49:54 2014 +0200
+++ b/vm-memory/agent/src/main/java/com/redhat/thermostat/vm/memory/agent/internal/VmMemoryVmListener.java	Mon Mar 10 18:35:14 2014 -0400
@@ -36,6 +36,8 @@
 
 package com.redhat.thermostat.vm.memory.agent.internal;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -55,6 +57,8 @@
     private final String vmId;
     private final VmMemoryStatDAO memDAO;
     private final String writerId;
+    
+    private boolean error;
 
     public VmMemoryVmListener(String writerId, VmMemoryStatDAO vmMemoryStatDao, String vmId) {
         memDAO = vmMemoryStatDao;
@@ -71,35 +75,121 @@
     void recordMemoryStat(VmMemoryDataExtractor extractor) {
         try {
             long timestamp = System.currentTimeMillis();
-            int maxGenerations = (int) extractor.getTotalGcGenerations();
-            Generation[] generations = new Generation[maxGenerations];
-            for (int generation = 0; generation < maxGenerations; generation++) {
-                Generation g = new Generation();
-                g.setName(extractor.getGenerationName(generation));
-                g.setCapacity(extractor.getGenerationCapacity(generation));
-                g.setMaxCapacity(extractor.getGenerationMaxCapacity(generation));
-                g.setCollector(extractor.getGenerationCollector(generation));
-                generations[generation] = g;
-                int maxSpaces = (int) extractor.getTotalSpaces(generation);
-                Space[] spaces = new Space[maxSpaces];
-                for (int space = 0; space < maxSpaces; space++) {
-                    Space s = new Space();
-                    s.setIndex((int) space);
-                    s.setName(extractor.getSpaceName(generation, space));
-                    s.setCapacity(extractor.getSpaceCapacity(generation, space));
-                    s.setMaxCapacity(extractor.getSpaceMaxCapacity(generation, space));
-                    s.setUsed(extractor.getSpaceUsed(generation, space));
-                    spaces[space] = s;
+            Long maxGenerations = extractor.getTotalGcGenerations();
+            if (maxGenerations != null) {
+                List<Generation> generations = new ArrayList<Generation>(maxGenerations.intValue());
+                for (int generation = 0; generation < maxGenerations; generation++) {
+                    Generation g = createGeneration(extractor, generation);
+                    if (g != null) {
+                        Long maxSpaces = extractor.getTotalSpaces(generation);
+                        if (maxSpaces != null) {
+                            List<Space> spaces = new ArrayList<Space>(maxSpaces.intValue());
+                            for (int space = 0; space < maxSpaces; space++) {
+                                Space s = createSpace(extractor, generation,
+                                        space);
+                                if (s != null) {
+                                    spaces.add(s);
+                                }
+                            }
+                            g.setSpaces(spaces.toArray(new Space[spaces.size()]));
+                            generations.add(g);
+                        }
+                        else {
+                            logWarningOnce("Unable to determine number of spaces in generation "
+                                    + generation + " for VM " + vmId + ". Skipping generation");
+                        }
+                    }
                 }
-                g.setSpaces(spaces);
+                VmMemoryStat stat = new VmMemoryStat(writerId, timestamp, vmId, 
+                        generations.toArray(new Generation[generations.size()]));
+                memDAO.putVmMemoryStat(stat);
             }
-            VmMemoryStat stat = new VmMemoryStat(writerId, timestamp, vmId, generations);
-            memDAO.putVmMemoryStat(stat);
+            else {
+                logWarningOnce("Unable to determine number of generations for VM " + vmId);
+            }
         } catch (VmUpdateException e) {
-            logger.log(Level.WARNING, "error gathering memory info for vm " + vmId, e);
+            logger.log(Level.WARNING, "Error gathering memory info for VM " + vmId, e);
         }
     }
 
+    private Generation createGeneration(VmMemoryDataExtractor extractor,
+            int generation) throws VmUpdateException {
+        String name = extractor.getGenerationName(generation);
+        if (name == null) {
+            logWarningOnce("Unable to determine name of generation " 
+                    + generation + " for VM " + vmId);
+            return null;
+        }
+        Long capacity = extractor.getGenerationCapacity(generation);
+        if (capacity == null) {
+            logWarningOnce("Unable to determine capacity of generation " 
+                    + generation + " for VM " + vmId);
+            return null;
+        }
+        Long maxCapacity = extractor.getGenerationMaxCapacity(generation);
+        if (maxCapacity == null) {
+            logWarningOnce("Unable to determine max capacity of generation " 
+                    + generation + " for VM " + vmId);
+            return null;
+        }
+        String collector = extractor.getGenerationCollector(generation);
+        if (collector == null) {
+            logWarningOnce("Unable to determine collector of generation " 
+                    + generation + " for VM " + vmId);
+            return null;
+        }
+        
+        Generation g = new Generation();
+        g.setName(name);
+        g.setCapacity(capacity);
+        g.setMaxCapacity(maxCapacity);
+        g.setCollector(collector);
+        return g;
+    }
+
+    private Space createSpace(VmMemoryDataExtractor extractor, int generation,
+            int space) throws VmUpdateException {
+        String name = extractor.getSpaceName(generation, space);
+        if (name == null) {
+            logWarningOnce("Unable to determine name of space " + space 
+                    + " in generation " + generation + " for VM " + vmId);
+            return null;
+        }
+        Long capacity = extractor.getSpaceCapacity(generation, space);
+        if (capacity == null) {
+            logWarningOnce("Unable to determine capacity of space " + space 
+                    + " in generation " + generation + " for VM " + vmId);
+            return null;
+        }
+        Long maxCapacity = extractor.getSpaceMaxCapacity(generation, space);
+        if (maxCapacity == null) {
+            logWarningOnce("Unable to determine max capacity of space " + space 
+                    + " in generation " + generation + " for VM " + vmId);
+            return null;
+        }
+        Long used = extractor.getSpaceUsed(generation, space);
+        if (used == null) {
+            logWarningOnce("Unable to determine used memory of space " + space 
+                    + " in generation " + generation + " for VM " + vmId);
+            return null;
+        }
+        
+        Space s = new Space();
+        s.setIndex(space);
+        s.setName(name);
+        s.setCapacity(capacity);
+        s.setMaxCapacity(maxCapacity);
+        s.setUsed(used);
+        return s;
+    }
+    
+    private void logWarningOnce(String message) {
+        if (!error) {
+            logger.log(Level.WARNING, message);
+            logger.log(Level.WARNING, "Further warnings will be ignored");
+            error = true;
+        }
+    }
 
 }
 
--- a/vm-memory/agent/src/test/java/com/redhat/thermostat/vm/memory/agent/internal/VmMemoryVmListenerTest.java	Thu May 22 10:49:54 2014 +0200
+++ b/vm-memory/agent/src/test/java/com/redhat/thermostat/vm/memory/agent/internal/VmMemoryVmListenerTest.java	Mon Mar 10 18:35:14 2014 -0400
@@ -37,8 +37,10 @@
 package com.redhat.thermostat.vm.memory.agent.internal;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.isA;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
@@ -188,5 +190,278 @@
 
         verifyNoMoreInteractions(vmMemoryStatDAO);
     }
+    
+    @Test
+    public void testRecordMemoryStatNoTotal() throws VmUpdateException {
+        when(extractor.getTotalGcGenerations()).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        verify(vmMemoryStatDAO, never()).putVmMemoryStat(any(VmMemoryStat.class));
+    }
+
+    @Test
+    public void testRecordMemoryStatNoName() throws VmUpdateException {
+        when(extractor.getGenerationName(0)).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        ArgumentCaptor<VmMemoryStat> captor = ArgumentCaptor.forClass(VmMemoryStat.class);
+        verify(vmMemoryStatDAO).putVmMemoryStat(captor.capture());
+        VmMemoryStat memoryStat = captor.getValue();
+        
+        Generation[] gens = memoryStat.getGenerations();
+        assertEquals(1, gens.length);
+        Generation gen = gens[0];
+        assertEquals(GEN_NAMES[1], gen.getName());
+        assertEquals(GEN_CAPS[1], (Long) gen.getCapacity());
+        assertEquals(GEN_MAX_CAPS[1], (Long) gen.getMaxCapacity());
+        assertEquals(GEN_GCS[1], gen.getCollector());
+        assertEquals(GEN_SPACES[1], Long.valueOf(gen.getSpaces().length));
+        Space[] spaces = gen.getSpaces();
+        for (int j = 1; j < spaces.length; j++) {
+            Space space = spaces[j];
+            assertEquals(SPACE_NAME[1][j], space.getName());
+            assertEquals(SPACE_CAPS[1][j], (Long) space.getCapacity());
+            assertEquals(SPACE_MAX_CAPS[1][j], (Long) space.getMaxCapacity());
+            assertEquals(SPACE_USED[1][j], (Long) space.getUsed());
+        }
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoCapacity() throws VmUpdateException {
+        when(extractor.getGenerationCapacity(0)).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        ArgumentCaptor<VmMemoryStat> captor = ArgumentCaptor.forClass(VmMemoryStat.class);
+        verify(vmMemoryStatDAO).putVmMemoryStat(captor.capture());
+        VmMemoryStat memoryStat = captor.getValue();
+        
+        Generation[] gens = memoryStat.getGenerations();
+        assertEquals(1, gens.length);
+        Generation gen = gens[0];
+        assertEquals(GEN_NAMES[1], gen.getName());
+        assertEquals(GEN_CAPS[1], (Long) gen.getCapacity());
+        assertEquals(GEN_MAX_CAPS[1], (Long) gen.getMaxCapacity());
+        assertEquals(GEN_GCS[1], gen.getCollector());
+        assertEquals(GEN_SPACES[1], Long.valueOf(gen.getSpaces().length));
+        Space[] spaces = gen.getSpaces();
+        for (int j = 1; j < spaces.length; j++) {
+            Space space = spaces[j];
+            assertEquals(SPACE_NAME[1][j], space.getName());
+            assertEquals(SPACE_CAPS[1][j], (Long) space.getCapacity());
+            assertEquals(SPACE_MAX_CAPS[1][j], (Long) space.getMaxCapacity());
+            assertEquals(SPACE_USED[1][j], (Long) space.getUsed());
+        }
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoMaxCapacity() throws VmUpdateException {
+        when(extractor.getGenerationMaxCapacity(0)).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        ArgumentCaptor<VmMemoryStat> captor = ArgumentCaptor.forClass(VmMemoryStat.class);
+        verify(vmMemoryStatDAO).putVmMemoryStat(captor.capture());
+        VmMemoryStat memoryStat = captor.getValue();
+        
+        Generation[] gens = memoryStat.getGenerations();
+        assertEquals(1, gens.length);
+        Generation gen = gens[0];
+        assertEquals(GEN_NAMES[1], gen.getName());
+        assertEquals(GEN_CAPS[1], (Long) gen.getCapacity());
+        assertEquals(GEN_MAX_CAPS[1], (Long) gen.getMaxCapacity());
+        assertEquals(GEN_GCS[1], gen.getCollector());
+        assertEquals(GEN_SPACES[1], Long.valueOf(gen.getSpaces().length));
+        Space[] spaces = gen.getSpaces();
+        for (int j = 1; j < spaces.length; j++) {
+            Space space = spaces[j];
+            assertEquals(SPACE_NAME[1][j], space.getName());
+            assertEquals(SPACE_CAPS[1][j], (Long) space.getCapacity());
+            assertEquals(SPACE_MAX_CAPS[1][j], (Long) space.getMaxCapacity());
+            assertEquals(SPACE_USED[1][j], (Long) space.getUsed());
+        }
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoCollector() throws VmUpdateException {
+        when(extractor.getGenerationCollector(0)).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        ArgumentCaptor<VmMemoryStat> captor = ArgumentCaptor.forClass(VmMemoryStat.class);
+        verify(vmMemoryStatDAO).putVmMemoryStat(captor.capture());
+        VmMemoryStat memoryStat = captor.getValue();
+        
+        Generation[] gens = memoryStat.getGenerations();
+        assertEquals(1, gens.length);
+        Generation gen = gens[0];
+        assertEquals(GEN_NAMES[1], gen.getName());
+        assertEquals(GEN_CAPS[1], (Long) gen.getCapacity());
+        assertEquals(GEN_MAX_CAPS[1], (Long) gen.getMaxCapacity());
+        assertEquals(GEN_GCS[1], gen.getCollector());
+        assertEquals(GEN_SPACES[1], Long.valueOf(gen.getSpaces().length));
+        Space[] spaces = gen.getSpaces();
+        for (int j = 1; j < spaces.length; j++) {
+            Space space = spaces[j];
+            assertEquals(SPACE_NAME[1][j], space.getName());
+            assertEquals(SPACE_CAPS[1][j], (Long) space.getCapacity());
+            assertEquals(SPACE_MAX_CAPS[1][j], (Long) space.getMaxCapacity());
+            assertEquals(SPACE_USED[1][j], (Long) space.getUsed());
+        }
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoTotalSpaces() throws VmUpdateException {
+        when(extractor.getTotalSpaces(0)).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        ArgumentCaptor<VmMemoryStat> captor = ArgumentCaptor.forClass(VmMemoryStat.class);
+        verify(vmMemoryStatDAO).putVmMemoryStat(captor.capture());
+        VmMemoryStat memoryStat = captor.getValue();
+        
+        Generation[] gens = memoryStat.getGenerations();
+        assertEquals(1, gens.length);
+        Generation gen = gens[0];
+        assertEquals(GEN_NAMES[1], gen.getName());
+        assertEquals(GEN_CAPS[1], (Long) gen.getCapacity());
+        assertEquals(GEN_MAX_CAPS[1], (Long) gen.getMaxCapacity());
+        assertEquals(GEN_GCS[1], gen.getCollector());
+        assertEquals(GEN_SPACES[1], Long.valueOf(gen.getSpaces().length));
+        Space[] spaces = gen.getSpaces();
+        for (int j = 1; j < spaces.length; j++) {
+            Space space = spaces[j];
+            assertEquals(SPACE_NAME[1][j], space.getName());
+            assertEquals(SPACE_CAPS[1][j], (Long) space.getCapacity());
+            assertEquals(SPACE_MAX_CAPS[1][j], (Long) space.getMaxCapacity());
+            assertEquals(SPACE_USED[1][j], (Long) space.getUsed());
+        }
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoSpaceName() throws VmUpdateException {
+        when(extractor.getSpaceName(0, 1)).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        ArgumentCaptor<VmMemoryStat> captor = ArgumentCaptor.forClass(VmMemoryStat.class);
+        verify(vmMemoryStatDAO).putVmMemoryStat(captor.capture());
+        VmMemoryStat memoryStat = captor.getValue();
+        
+        Generation[] gens = memoryStat.getGenerations();
+        assertEquals(2, gens.length);
+        for (int i = 0; i < gens.length; i++) {
+            Generation gen = gens[i];
+            assertEquals(GEN_NAMES[i], gen.getName());
+            assertEquals(GEN_CAPS[i], (Long) gen.getCapacity());
+            assertEquals(GEN_MAX_CAPS[i], (Long) gen.getMaxCapacity());
+            assertEquals(GEN_GCS[i], gen.getCollector());
+            if (i == 0) {
+                // Bad space in first generation
+                assertEquals(Long.valueOf(GEN_SPACES[i] - 1), Long.valueOf(gen.getSpaces().length));
+            }
+            else {
+                assertEquals(GEN_SPACES[i], Long.valueOf(gen.getSpaces().length));
+            }
+            Space[] spaces = gen.getSpaces();
+            for (int j = 0; j < spaces.length; j++) {
+                Space space = spaces[j];
+                assertEquals(SPACE_NAME[i][j], space.getName());
+                assertEquals(SPACE_CAPS[i][j], (Long) space.getCapacity());
+                assertEquals(SPACE_MAX_CAPS[i][j], (Long) space.getMaxCapacity());
+                assertEquals(SPACE_USED[i][j], (Long) space.getUsed());
+            }
+        }
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoSpaceCapacity() throws VmUpdateException {
+        when(extractor.getSpaceCapacity(0, 1)).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        ArgumentCaptor<VmMemoryStat> captor = ArgumentCaptor.forClass(VmMemoryStat.class);
+        verify(vmMemoryStatDAO).putVmMemoryStat(captor.capture());
+        VmMemoryStat memoryStat = captor.getValue();
+        
+        Generation[] gens = memoryStat.getGenerations();
+        assertEquals(2, gens.length);
+        for (int i = 0; i < gens.length; i++) {
+            Generation gen = gens[i];
+            assertEquals(GEN_NAMES[i], gen.getName());
+            assertEquals(GEN_CAPS[i], (Long) gen.getCapacity());
+            assertEquals(GEN_MAX_CAPS[i], (Long) gen.getMaxCapacity());
+            assertEquals(GEN_GCS[i], gen.getCollector());
+            if (i == 0) {
+                // Bad space in first generation
+                assertEquals(Long.valueOf(GEN_SPACES[i] - 1), Long.valueOf(gen.getSpaces().length));
+            }
+            else {
+                assertEquals(GEN_SPACES[i], Long.valueOf(gen.getSpaces().length));
+            }
+            Space[] spaces = gen.getSpaces();
+            for (int j = 0; j < spaces.length; j++) {
+                Space space = spaces[j];
+                assertEquals(SPACE_NAME[i][j], space.getName());
+                assertEquals(SPACE_CAPS[i][j], (Long) space.getCapacity());
+                assertEquals(SPACE_MAX_CAPS[i][j], (Long) space.getMaxCapacity());
+                assertEquals(SPACE_USED[i][j], (Long) space.getUsed());
+            }
+        }
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoSpaceMaxCapacity() throws VmUpdateException {
+        when(extractor.getSpaceMaxCapacity(0, 1)).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        ArgumentCaptor<VmMemoryStat> captor = ArgumentCaptor.forClass(VmMemoryStat.class);
+        verify(vmMemoryStatDAO).putVmMemoryStat(captor.capture());
+        VmMemoryStat memoryStat = captor.getValue();
+        
+        Generation[] gens = memoryStat.getGenerations();
+        assertEquals(2, gens.length);
+        for (int i = 0; i < gens.length; i++) {
+            Generation gen = gens[i];
+            assertEquals(GEN_NAMES[i], gen.getName());
+            assertEquals(GEN_CAPS[i], (Long) gen.getCapacity());
+            assertEquals(GEN_MAX_CAPS[i], (Long) gen.getMaxCapacity());
+            assertEquals(GEN_GCS[i], gen.getCollector());
+            if (i == 0) {
+                // Bad space in first generation
+                assertEquals(Long.valueOf(GEN_SPACES[i] - 1), Long.valueOf(gen.getSpaces().length));
+            }
+            else {
+                assertEquals(GEN_SPACES[i], Long.valueOf(gen.getSpaces().length));
+            }
+            Space[] spaces = gen.getSpaces();
+            for (int j = 0; j < spaces.length; j++) {
+                Space space = spaces[j];
+                assertEquals(SPACE_NAME[i][j], space.getName());
+                assertEquals(SPACE_CAPS[i][j], (Long) space.getCapacity());
+                assertEquals(SPACE_MAX_CAPS[i][j], (Long) space.getMaxCapacity());
+                assertEquals(SPACE_USED[i][j], (Long) space.getUsed());
+            }
+        }
+    }
+    
+    @Test
+    public void testRecordMemoryStatNoSpaceUsed() throws VmUpdateException {
+        when(extractor.getSpaceUsed(0, 1)).thenReturn(null);
+        vmListener.recordMemoryStat(extractor);
+        ArgumentCaptor<VmMemoryStat> captor = ArgumentCaptor.forClass(VmMemoryStat.class);
+        verify(vmMemoryStatDAO).putVmMemoryStat(captor.capture());
+        VmMemoryStat memoryStat = captor.getValue();
+        
+        Generation[] gens = memoryStat.getGenerations();
+        assertEquals(2, gens.length);
+        for (int i = 0; i < gens.length; i++) {
+            Generation gen = gens[i];
+            assertEquals(GEN_NAMES[i], gen.getName());
+            assertEquals(GEN_CAPS[i], (Long) gen.getCapacity());
+            assertEquals(GEN_MAX_CAPS[i], (Long) gen.getMaxCapacity());
+            assertEquals(GEN_GCS[i], gen.getCollector());
+            if (i == 0) {
+                // Bad space in first generation
+                assertEquals(Long.valueOf(GEN_SPACES[i] - 1), Long.valueOf(gen.getSpaces().length));
+            }
+            else {
+                assertEquals(GEN_SPACES[i], Long.valueOf(gen.getSpaces().length));
+            }
+            Space[] spaces = gen.getSpaces();
+            for (int j = 0; j < spaces.length; j++) {
+                Space space = spaces[j];
+                assertEquals(SPACE_NAME[i][j], space.getName());
+                assertEquals(SPACE_CAPS[i][j], (Long) space.getCapacity());
+                assertEquals(SPACE_MAX_CAPS[i][j], (Long) space.getMaxCapacity());
+                assertEquals(SPACE_USED[i][j], (Long) space.getUsed());
+            }
+        }
+    }
 }