--- a/agent/core/src/main/java/com/redhat/thermostat/agent/Agent.java	Tue Oct 03 13:25:56 2017 -0400
+++ b/agent/core/src/main/java/com/redhat/thermostat/agent/Agent.java	Wed Oct 04 15:08:14 2017 -0400
@@ -45,6 +45,7 @@
 import com.redhat.thermostat.agent.dao.AgentInfoDAO;
 import com.redhat.thermostat.agent.dao.BackendInfoDAO;
 import com.redhat.thermostat.backend.Backend;
+import com.redhat.thermostat.backend.BackendActivator;
 import com.redhat.thermostat.backend.BackendRegistry;
 import com.redhat.thermostat.common.ActionEvent;
 import com.redhat.thermostat.common.ActionListener;
@@ -61,6 +62,9 @@
 public class Agent {
 
     private static final Logger logger = LoggingUtils.getLogger(Agent.class);
+    private static final int BACKEND_ACTIVATION_TIMEOUT_SECONDS = 65;
+    private static final int BACKEND_DEACTIVATION_TIMEOUT_SECONDS = 10;
+    private static final int BACKEND_ACTIVATOR_THREAD_COUNT = 2;
 
     private final BackendRegistry backendRegistry;
     private final AgentStartupConfiguration config;
@@ -68,6 +72,8 @@
     private final AgentInfoDAO agentDao;
     private final BackendInfoDAO backendDao;
     private final WriterID writerID;
+    private final BackendActivator backendActivator;
+    private final Object backendActivationLock = new Object();
     
     private AgentInformation agentInfo;
     private boolean started = false;
@@ -77,39 +83,39 @@
     {
         @Override
         public void actionPerformed(ActionEvent<ThermostatExtensionRegistry.Action> actionEvent) {
-            Backend backend = (Backend) actionEvent.getPayload();
+            final Backend backend = (Backend) actionEvent.getPayload();
             
             switch (actionEvent.getActionId()) {
-
                 case SERVICE_ADDED: {
                     if (!backendInfos.containsKey(backend)) {
-
-                        logger.info("Adding backend: " + backend);
-
-                        backend.activate();
-
-                        BackendInformation info = AgentHelper.createBackendInformation(backend, getId());
-                        backendDao.addBackendInformation(info);
-                        backendInfos.put(backend, info);                    
-                    } else {
-                        logger.warning("Backend registered that agent already knows about:" + backend);
+                        backendActivator.activateBackend(backend, new Runnable() {
+                            @Override
+                            public void run() {
+                                synchronized (backendActivationLock) {
+                                    BackendInformation info = AgentHelper.createBackendInformation(backend, getId());
+                                    backendDao.addBackendInformation(info);
+                                    backendInfos.put(backend, info);
+                                }
+                            }
+                        });
                     }
                     break;
                 }
-
                 case SERVICE_REMOVED: {
-                    BackendInformation info = backendInfos.get(backend);
+                    final BackendInformation info = backendInfos.get(backend);
                     if (info != null) {
-                        logger.info("removing backend: " + backend);
-
-                        backend.deactivate();
-
-                        backendDao.removeBackendInformation(info);
-                        backendInfos.remove(backend); 
+                        backendActivator.deactivateBackend(backend, new Runnable() {
+                            @Override
+                            public void run() {
+                                synchronized (backendActivationLock) {
+                                    backendDao.removeBackendInformation(info);
+                                    backendInfos.remove(backend);
+                                }
+                            }
+                        });
                     }
                     break;
                 }
-
                 default: {
                     logger.log(Level.WARNING, "received unknown event from BackendRegistry: " + actionEvent.getActionId());
                     break;
@@ -117,17 +123,23 @@
             }
         }
     };
-    
+
     public Agent(BackendRegistry registry, AgentStartupConfiguration config,
-            AgentInfoDAO agentInfoDao, BackendInfoDAO backendInfoDao, WriterID writerId) {
+                 AgentInfoDAO agentInfoDao, BackendInfoDAO backendInfoDao, WriterID writerId) {
+        this(registry, config, agentInfoDao, backendInfoDao, writerId,
+             new BackendActivator(BACKEND_ACTIVATOR_THREAD_COUNT, BACKEND_ACTIVATION_TIMEOUT_SECONDS));
+    }
+
+    Agent(BackendRegistry registry, AgentStartupConfiguration config, AgentInfoDAO agentInfoDao,
+          BackendInfoDAO backendInfoDao, WriterID writerId, BackendActivator backendActivator) {
         this.backendRegistry = registry;
         this.config = config;
         this.agentDao = agentInfoDao;
         this.backendDao = backendInfoDao;
         this.writerID = writerId;
-        
-        backendInfos = new ConcurrentHashMap<>();
-        
+        this.backendActivator = backendActivator;
+        this.backendInfos = new ConcurrentHashMap<>();
+
         backendRegistry.addActionListener(backendRegistryListener);
     }
 
@@ -156,10 +168,11 @@
         return agentInfo;
     }
 
-
     public synchronized void stop() {
         if (started) {
             backendRegistry.stop();
+            backendActivator.waitForAllBackendsToDeactivate(BACKEND_DEACTIVATION_TIMEOUT_SECONDS);
+            backendActivator.shutdown();
             
             if (config.purge()) {
                 removeAllAgentRelatedInformation();

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/agent/core/src/main/java/com/redhat/thermostat/backend/BackendActivator.java	Wed Oct 04 15:08:14 2017 -0400
@@ -0,0 +1,294 @@
+/*
+ * Copyright 2012-2017 Red Hat, Inc.
+ *
+ * This file is part of Thermostat.
+ *
+ * Thermostat is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; either version 2, or (at your
+ * option) any later version.
+ *
+ * Thermostat is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Thermostat; see the file COPYING.  If not see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Linking this code with other modules is making a combined work
+ * based on this code.  Thus, the terms and conditions of the GNU
+ * General Public License cover the whole combination.
+ *
+ * As a special exception, the copyright holders of this code give
+ * you permission to link this code with independent modules to
+ * produce an executable, regardless of the license terms of these
+ * independent modules, and to copy and distribute the resulting
+ * executable under terms of your choice, provided that you also
+ * meet, for each linked independent module, the terms and conditions
+ * of the license of that module.  An independent module is a module
+ * which is not derived from or based on this code.  If you modify
+ * this code, you may extend this exception to your version of the
+ * library, but you are not obligated to do so.  If you do not wish
+ * to do so, delete this exception statement from your version.
+ */
+
+package com.redhat.thermostat.backend;
+
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Logger;
+
+import com.redhat.thermostat.common.utils.LoggingUtils;
+
+/**
+ * An asynchronous backend activator, designed to process activate/deactivate
+ * calls such that any misbehaving backends will not block the other backends
+ * from activating or deactivating. This class is designed to be thread safe.
+ * Once no more activation and deactivation will be done, it should have the
+ * {@link #shutdown()} method called. Any operations requested from this object
+ * after it has shutdown will do nothing (and will print a warning message).
+ *
+ * Note: This implementation currently does not handle removing all backends,
+ * but not shutting down the agent itself.
+ */
+public class BackendActivator {
+
+    private static final Logger logger = LoggingUtils.getLogger(BackendActivator.class);
+    private static final int NO_SCHEDULE_DELAY_SECONDS = 0;
+    private static final int AMOUNT_OF_RESUBMISSIONS_TO_EXECUTOR = 4;
+    private static final int RESUBMISSION_DURATION_SECONDS = 15;
+    private static final int STARTING_BACKEND_TRACKER_ATTEMPT_NUMBER = 1;
+
+    private final long timeoutSeconds;
+    private final ScheduledExecutorService activationExecutorService;
+    private final ScheduledExecutorService deactivationExecutorService;
+    private final ScheduledExecutorService timeoutExecutorService;
+    private final ExecutorService postActivateService;
+    private final CountDownLatch allBackendsDeactivated;
+    private AtomicBoolean shutdown = new AtomicBoolean(false);
+    private AtomicInteger numBackendsActivate = new AtomicInteger(0);
+
+    /**
+     * Creates a new backend activator with the provided number of threads and
+     * timeout to (attempt to) kill misbehaving backends. This will generate
+     * a total of (2 * numBackendsInParallel) + 2 threads.
+     * @param numBackendsInParallel How many threads this should allocate to
+     *                              activating/deactivating backends.
+     * @param timeoutSeconds How many seconds before a backend is considered to
+     *                       be misbehaving and should attempt to interrupt it.
+     *                       This should be non-negative.
+     * @throws IllegalArgumentException If the timeout is negative.
+     */
+    public BackendActivator(int numBackendsInParallel, long timeoutSeconds) {
+        this(timeoutSeconds, Executors.newScheduledThreadPool(numBackendsInParallel),
+                Executors.newScheduledThreadPool(numBackendsInParallel), Executors.newScheduledThreadPool(1),
+                Executors.newFixedThreadPool(1), new CountDownLatch(1));
+    }
+
+    BackendActivator(long timeoutSeconds, ScheduledExecutorService activationExecutorService,
+                     ScheduledExecutorService deactivationExecutorService, ScheduledExecutorService timeoutExecutorService,
+                     ExecutorService postActivateService, CountDownLatch allBackendsDeactivated) {
+        if (timeoutSeconds < 0) {
+            throw new IllegalArgumentException("Cannot have a negative backend activation timeout");
+        }
+
+        this.timeoutSeconds = timeoutSeconds;
+        this.activationExecutorService = Objects.requireNonNull(activationExecutorService);
+        this.deactivationExecutorService = Objects.requireNonNull(deactivationExecutorService);
+        this.timeoutExecutorService = Objects.requireNonNull(timeoutExecutorService);
+        this.postActivateService = Objects.requireNonNull(postActivateService);
+        this.allBackendsDeactivated = Objects.requireNonNull(allBackendsDeactivated);
+    }
+
+    /**
+     * Keeps recursively scheduling the backend tracker every period of
+     * RESUBMISSION_DURATION_SECONDS. Termination of recursion occurs when the
+     * attempt number is greater than AMOUNT_OF_RESUBMISSIONS_TO_EXECUTOR.
+     */
+    private void scheduleBackendTrackerRecursively(final BackendTracker backendTracker) {
+        timeoutExecutorService.schedule(new Runnable() {
+            @Override
+            public void run() {
+                if (!backendTracker.getFuture().isDone() && backendTracker.getAttemptNumber() <= AMOUNT_OF_RESUBMISSIONS_TO_EXECUTOR) {
+                    String logMessage = String.format("Backend '%s' is taking a while to activate/deactivate [%d/%d]",
+                                                      backendTracker.getBackend(), backendTracker.getAttemptNumber(),
+                                                      AMOUNT_OF_RESUBMISSIONS_TO_EXECUTOR);
+                    logger.warning(logMessage);
+                    backendTracker.incrementAttemptNumber();
+                    scheduleBackendTrackerRecursively(backendTracker);
+                }
+            }
+        }, RESUBMISSION_DURATION_SECONDS, TimeUnit.SECONDS);
+    }
+
+    private void scheduleAction(final Backend backend, final boolean isActivating, Runnable runnable) {
+        ScheduledExecutorService executor = (isActivating ? activationExecutorService : deactivationExecutorService);
+        final ScheduledFuture<?> future = executor.schedule(runnable, NO_SCHEDULE_DELAY_SECONDS, TimeUnit.SECONDS);
+
+        timeoutExecutorService.schedule(new Runnable() {
+            @Override
+            public void run() {
+                if (!future.isDone()) {
+                    logger.warning("Backend '" + backend + "' is taking too long to " + (isActivating ? "activate" : "deactivate"));
+                    future.cancel(true);
+                }
+            }
+        }, timeoutSeconds, TimeUnit.SECONDS);
+
+        BackendTracker backendTracker = new BackendTracker(backend, future, STARTING_BACKEND_TRACKER_ATTEMPT_NUMBER);
+        scheduleBackendTrackerRecursively(backendTracker);
+    }
+
+    /**
+     * Activates a backend. Will attempt to kill activation if it takes longer
+     * than the value set in the constructor for timing out.
+     * @param backend The backend to activate, should not be null.
+     * @param postActivate Any actions to be performed after activation. This
+     *                     may be null if you do not want to run anything after
+     *                     activation.
+     * @throws NullPointerException If the backend is null.
+     */
+    public void activateBackend(final Backend backend, final Runnable postActivate) {
+        Objects.requireNonNull(backend);
+
+        if (shutdown.get()) {
+            logger.info("Cannot activate '" + backend + "', backend activator shut down");
+            return;
+        }
+
+        logger.info("Adding backend: " + backend);
+        scheduleAction(backend, true, new Runnable() {
+            @Override
+            public void run() {
+                final boolean didActivate = backend.activate();
+                if (!didActivate) {
+                    logger.warning("Backend '" + backend + "' failed to activate.");
+                }
+                if (postActivate != null && didActivate) {
+                    postActivateService.submit(postActivate);
+                }
+                numBackendsActivate.incrementAndGet();
+            }
+        });
+    }
+
+    /**
+     * Deactivates a backend. Will attempt to kill it if it takes longer than
+     * the value set in the constructor for timing out.
+     * @param backend The backend to activate, should not be null.
+     * @param postDeactivate Any actions to be performed after activation. This
+     *                       may be null if you do not want to run anything
+     *                       after deactivation.
+     * @throws NullPointerException If the backend is null.
+     */
+    public void deactivateBackend(final Backend backend, final Runnable postDeactivate) {
+        Objects.requireNonNull(backend);
+
+        if (shutdown.get()) {
+            logger.info("Cannot deactivate '" + backend + "', backend activator shut down");
+            return;
+        }
+
+        logger.info("Removing backend: " + backend);
+        scheduleAction(backend, false, new Runnable() {
+            @Override
+            public void run() {
+                if (!backend.deactivate()) {
+                    logger.warning("Backend '" + backend + "' failed to deactivate.");
+                    return;
+                }
+                if (postDeactivate != null) {
+                    postActivateService.submit(postDeactivate);
+                }
+                if (numBackendsActivate.decrementAndGet() == 0) {
+                    allBackendsDeactivated.countDown();
+                }
+            }
+        });
+    }
+
+    /**
+     * Waits for backends to deactivate. This depends on how many backends you
+     * have registered with this object, as it counts the number activated and
+     * deactivated, and will either return when the net change of activation to
+     * deactivation is zero, or if it times out based on the provided argument.
+     * @param timeoutSeconds How long until this method will return. If this is
+     *                       zero of negative, it will return immediately as
+     *                       stated in the CountDownLatch specification.
+     * @return True if there was no timeout, false otherwise.
+     */
+    public boolean waitForAllBackendsToDeactivate(int timeoutSeconds) {
+        try {
+            if (allBackendsDeactivated.await(timeoutSeconds, TimeUnit.SECONDS)) {
+                return true;
+            } else {
+                logger.warning("Timed out waiting for backends to deactivate");
+            }
+        } catch (InterruptedException e) {
+            logger.warning("Waiting for deactivation latch interrupted");
+        }
+
+        return false;
+    }
+
+    /**
+     * Shuts down the backend activator so it can no longer accept any backends
+     * for activation or deactivation.
+     */
+    public void shutdown() {
+        if (shutdown.get()) {
+            logger.warning("Already shut down the backend activator");
+            return;
+        }
+
+        activationExecutorService.shutdownNow();
+        deactivationExecutorService.shutdownNow();
+        timeoutExecutorService.shutdownNow();
+        postActivateService.shutdownNow();
+
+        shutdown.set(true);
+    }
+
+    // For unit testing.
+    int getNumberOfBackendsActive() {
+        return numBackendsActivate.get();
+    }
+
+    static class BackendTracker {
+        private Backend backend;
+        private Future<?> future;
+        private int attemptNumber;
+
+        BackendTracker(Backend backend, Future<?> future, int attemptNumber) {
+            this.backend = Objects.requireNonNull(backend);
+            this.future = Objects.requireNonNull(future);
+            this.attemptNumber = attemptNumber;
+        }
+
+        void incrementAttemptNumber() {
+            attemptNumber++;
+        }
+
+        Backend getBackend() {
+            return backend;
+        }
+
+        Future<?> getFuture() {
+            return future;
+        }
+
+        int getAttemptNumber() {
+            return attemptNumber;
+        }
+    }
+}

--- a/agent/core/src/test/java/com/redhat/thermostat/agent/AgentTest.java	Tue Oct 03 13:25:56 2017 -0400
+++ b/agent/core/src/test/java/com/redhat/thermostat/agent/AgentTest.java	Wed Oct 04 15:08:14 2017 -0400
@@ -48,6 +48,7 @@
 
 import java.util.UUID;
 
+import com.redhat.thermostat.backend.BackendActivator;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
@@ -70,9 +71,9 @@
     private AgentStartupConfiguration config;
     private BackendRegistry backendRegistry;
     private Backend backend;
-
     private AgentInfoDAO agentInfoDao;
     private BackendInfoDAO backendInfoDao;
+    private BackendActivator backendActivator;
     
     @Before
     public void setUp() throws Exception {
@@ -91,6 +92,7 @@
         when(backend.isActive()).thenReturn(true);
 
         backendRegistry = mock(BackendRegistry.class);
+        backendActivator = new InstantBackendActivator();
     }
     
     @SuppressWarnings("unused")
@@ -108,7 +110,7 @@
         UUID uuid = UUID.randomUUID();
         WriterID id = mock(WriterID.class);
         when(id.getWriterID()).thenReturn(uuid.toString());
-        Agent agent = new Agent(backendRegistry, config, agentInfoDao, backendInfoDao, id);
+        Agent agent = new Agent(backendRegistry, config, agentInfoDao, backendInfoDao, id, backendActivator);
         
         agent.start();
 
@@ -127,7 +129,7 @@
 
         // Start agent.
         WriterID id = mock(WriterID.class);
-        Agent agent = new Agent(backendRegistry, config, agentInfoDao, backendInfoDao, id);
+        Agent agent = new Agent(backendRegistry, config, agentInfoDao, backendInfoDao, id, backendActivator);
         verify(backendRegistry).addActionListener(backendListener.capture());
         
         agent.start();
@@ -207,5 +209,24 @@
         verify(agentInfoDao).updateAgentInformation(isA(AgentInformation.class));
         //verify(storage, times(0)).purge(anyString()); TODO
     }
+
+    static class InstantBackendActivator extends BackendActivator {
+
+        public InstantBackendActivator() {
+            super(1, 1);
+        }
+
+        @Override
+        public void activateBackend(final Backend backend, final Runnable postActivate) {
+            backend.activate();
+            postActivate.run();
+        }
+
+        @Override
+        public void deactivateBackend(final Backend backend, final Runnable postDeactivate) {
+            backend.deactivate();
+            postDeactivate.run();
+        }
+    }
 }
 

--- a/agent/core/src/test/java/com/redhat/thermostat/agent/internal/AgentApplicationTest.java	Tue Oct 03 13:25:56 2017 -0400
+++ b/agent/core/src/test/java/com/redhat/thermostat/agent/internal/AgentApplicationTest.java	Wed Oct 04 15:08:14 2017 -0400
@@ -36,6 +36,7 @@
 
 package com.redhat.thermostat.agent.internal;
 
+import static com.redhat.thermostat.testutils.Asserts.assertCommandIsRegistered;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -54,6 +55,22 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import com.redhat.thermostat.agent.Agent;
+import com.redhat.thermostat.agent.config.AgentStartupConfiguration;
+import com.redhat.thermostat.agent.dao.AgentInfoDAO;
+import com.redhat.thermostat.agent.dao.BackendInfoDAO;
+import com.redhat.thermostat.agent.internal.AgentApplication.ConfigurationCreator;
+import com.redhat.thermostat.backend.BackendRegistry;
+import com.redhat.thermostat.common.ExitStatus;
+import com.redhat.thermostat.common.LaunchException;
+import com.redhat.thermostat.common.Version;
+import com.redhat.thermostat.common.cli.Arguments;
+import com.redhat.thermostat.common.cli.CommandContext;
+import com.redhat.thermostat.common.cli.CommandException;
+import com.redhat.thermostat.shared.config.CommonPaths;
+import com.redhat.thermostat.shared.config.InvalidConfigurationException;
+import com.redhat.thermostat.storage.core.WriterID;
+import com.redhat.thermostat.testutils.StubBundleContext;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -67,25 +84,6 @@
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
-import static com.redhat.thermostat.testutils.Asserts.assertCommandIsRegistered;
-
-import com.redhat.thermostat.agent.Agent;
-import com.redhat.thermostat.agent.internal.AgentApplication.ConfigurationCreator;
-import com.redhat.thermostat.agent.config.AgentStartupConfiguration;
-import com.redhat.thermostat.agent.dao.AgentInfoDAO;
-import com.redhat.thermostat.agent.dao.BackendInfoDAO;
-import com.redhat.thermostat.backend.BackendRegistry;
-import com.redhat.thermostat.common.ExitStatus;
-import com.redhat.thermostat.common.LaunchException;
-import com.redhat.thermostat.common.Version;
-import com.redhat.thermostat.common.cli.Arguments;
-import com.redhat.thermostat.common.cli.CommandContext;
-import com.redhat.thermostat.common.cli.CommandException;
-import com.redhat.thermostat.shared.config.CommonPaths;
-import com.redhat.thermostat.shared.config.InvalidConfigurationException;
-import com.redhat.thermostat.storage.core.WriterID;
-import com.redhat.thermostat.testutils.StubBundleContext;
-
 @RunWith(PowerMockRunner.class)
 public class AgentApplicationTest {
 

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/agent/core/src/test/java/com/redhat/thermostat/backend/BackendActivatorTest.java	Wed Oct 04 15:08:14 2017 -0400
@@ -0,0 +1,277 @@
+/*
+ * Copyright 2012-2017 Red Hat, Inc.
+ *
+ * This file is part of Thermostat.
+ *
+ * Thermostat is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; either version 2, or (at your
+ * option) any later version.
+ *
+ * Thermostat is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Thermostat; see the file COPYING.  If not see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Linking this code with other modules is making a combined work
+ * based on this code.  Thus, the terms and conditions of the GNU
+ * General Public License cover the whole combination.
+ *
+ * As a special exception, the copyright holders of this code give
+ * you permission to link this code with independent modules to
+ * produce an executable, regardless of the license terms of these
+ * independent modules, and to copy and distribute the resulting
+ * executable under terms of your choice, provided that you also
+ * meet, for each linked independent module, the terms and conditions
+ * of the license of that module.  An independent module is a module
+ * which is not derived from or based on this code.  If you modify
+ * this code, you may extend this exception to your version of the
+ * library, but you are not obligated to do so.  If you do not wish
+ * to do so, delete this exception statement from your version.
+ */
+
+package com.redhat.thermostat.backend;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import com.redhat.thermostat.backend.adverse.AdverseBackend;
+import com.redhat.thermostat.backend.adverse.AdverseBackendActivation;
+import com.redhat.thermostat.backend.adverse.AdverseBackendDeactivation;
+import org.junit.Test;
+
+public class BackendActivatorTest {
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testDoesNotAcceptNegativeArguments() {
+        new BackendActivator(-5, -2);
+    }
+
+    @Test
+    public void testActivateBackend() {
+        Backend backend = mock(Backend.class);
+        when(backend.activate()).thenReturn(true);
+        Runnable postActivateRunnable = mock(Runnable.class);
+        ScheduledExecutorService scheduledExecutorService = new InstantScheduledThreadPoolExecutor();
+        ExecutorService executorService = new InstantExecutorService();
+        BackendActivator backendActivator = new BackendActivator(3, scheduledExecutorService, scheduledExecutorService,
+                                                                 scheduledExecutorService, executorService, new CountDownLatch(1));
+
+        assertEquals(0, backendActivator.getNumberOfBackendsActive());
+        backendActivator.activateBackend(backend, postActivateRunnable);
+        assertEquals(1, backendActivator.getNumberOfBackendsActive());
+
+        verify(backend, times(1)).activate();
+        verify(postActivateRunnable, times(1)).run();
+    }
+
+    @Test
+    public void testDeactivateBackend() {
+        Backend backend = mock(Backend.class);
+        when(backend.deactivate()).thenReturn(true);
+        Runnable postDeactivateRunnable = mock(Runnable.class);
+        ScheduledExecutorService scheduledExecutorService = new InstantScheduledThreadPoolExecutor();
+        ExecutorService executorService = new InstantExecutorService();
+        BackendActivator backendActivator = new BackendActivator(3, scheduledExecutorService, scheduledExecutorService,
+                                                                 scheduledExecutorService, executorService, new CountDownLatch(1));
+        backendActivator.deactivateBackend(backend, postDeactivateRunnable);
+
+        verify(backend, times(1)).deactivate();
+        verify(postDeactivateRunnable, times(1)).run();
+    }
+
+    @Test
+    public void testActivateBackendNotTrackedOrPostActivated() {
+        Backend backend = mock(Backend.class);
+        when(backend.activate()).thenReturn(false);
+        Runnable postActivateRunnable = mock(Runnable.class);
+        ScheduledExecutorService scheduledExecutorService = new InstantScheduledThreadPoolExecutor();
+        ExecutorService executorService = new InstantExecutorService();
+        BackendActivator backendActivator = new BackendActivator(3, scheduledExecutorService, scheduledExecutorService,
+                scheduledExecutorService, executorService, new CountDownLatch(1));
+        backendActivator.activateBackend(backend, postActivateRunnable);
+
+        verify(backend, times(1)).activate();
+        verify(postActivateRunnable, times(0)).run();
+        assertEquals(1, backendActivator.getNumberOfBackendsActive());
+    }
+
+    @Test
+    public void testDeactivateBackendNotTrackedOrPostDeactivated() {
+        Backend backend = mock(Backend.class);
+        when(backend.deactivate()).thenReturn(false);
+        Runnable postDeactivateRunnable = mock(Runnable.class);
+        ScheduledExecutorService scheduledExecutorService = new InstantScheduledThreadPoolExecutor();
+        ExecutorService executorService = new InstantExecutorService();
+        BackendActivator backendActivator = new BackendActivator(3, scheduledExecutorService, scheduledExecutorService,
+                scheduledExecutorService, executorService, new CountDownLatch(1));
+        backendActivator.deactivateBackend(backend, postDeactivateRunnable);
+
+        verify(backend, times(1)).deactivate();
+        verify(postDeactivateRunnable, times(0)).run();
+        assertEquals(0, backendActivator.getNumberOfBackendsActive());
+    }
+
+    @Test
+    public void testActivateAndDeactivate() {
+        Backend backend = mock(Backend.class);
+        when(backend.activate()).thenReturn(true);
+        when(backend.deactivate()).thenReturn(true);
+        Runnable postActivateRunnable = mock(Runnable.class);
+        Runnable postDeactivateRunnable = mock(Runnable.class);
+        ScheduledExecutorService scheduledExecutorService = new InstantScheduledThreadPoolExecutor();
+        ExecutorService executorService = new InstantExecutorService();
+        BackendActivator backendActivator = new BackendActivator(3, scheduledExecutorService, scheduledExecutorService,
+                                                                 scheduledExecutorService, executorService, new CountDownLatch(1));
+
+        assertEquals(0, backendActivator.getNumberOfBackendsActive());
+        backendActivator.activateBackend(backend, postActivateRunnable);
+        assertEquals(1, backendActivator.getNumberOfBackendsActive());
+        backendActivator.deactivateBackend(backend, postDeactivateRunnable);
+        assertEquals(0, backendActivator.getNumberOfBackendsActive());
+
+        verify(backend, times(1)).activate();
+        verify(backend, times(1)).deactivate();
+        verify(postActivateRunnable, times(1)).run();
+        verify(postDeactivateRunnable, times(1)).run();
+    }
+
+    @Test(timeout = 3000)
+    public void testAdverseActivation() {
+        AdverseBackend backend = new AdverseBackendActivation();
+        Runnable postDeactivateRunnable = mock(Runnable.class);
+
+        BackendActivator backendActivator = new BackendActivator(2, 1);
+        backendActivator.activateBackend(backend, postDeactivateRunnable);
+        backend.waitUntilInterrupted();
+    }
+
+    @Test(timeout = 3000)
+    public void testAdverseDeactivation() {
+        AdverseBackend backend = new AdverseBackendDeactivation();
+        Runnable postDeactivateRunnable = mock(Runnable.class);
+
+        BackendActivator backendActivator = new BackendActivator(2, 1);
+        backendActivator.deactivateBackend(backend, postDeactivateRunnable);
+        backend.waitUntilInterrupted();
+    }
+
+    @Test(timeout = 1000)
+    public void testCountsDownBackend() {
+        CountDownLatch countDownLatch = mock(CountDownLatch.class);
+        Backend backend = mock(Backend.class);
+        when(backend.activate()).thenReturn(true);
+        when(backend.deactivate()).thenReturn(true);
+        ScheduledExecutorService scheduledExecutorService = new InstantScheduledThreadPoolExecutor();
+        ExecutorService executorService = new InstantExecutorService();
+        BackendActivator backendActivator = new BackendActivator(1, scheduledExecutorService, scheduledExecutorService,
+                                                                 scheduledExecutorService, executorService, countDownLatch);
+
+        verify(countDownLatch, times(0)).countDown();
+        backendActivator.activateBackend(backend, mock(Runnable.class));
+        verify(countDownLatch, times(0)).countDown();
+
+        backendActivator.deactivateBackend(backend, mock(Runnable.class));
+        verify(countDownLatch, times(1)).countDown();
+    }
+
+    @Test(timeout = 1000)
+    public void testWaitsForDeactivation() {
+        CountDownLatch countDownLatch = new CountDownLatch(1);
+        Backend backend = mock(Backend.class);
+        when(backend.activate()).thenReturn(true);
+        when(backend.deactivate()).thenReturn(true);
+        ScheduledExecutorService scheduledExecutorService = new InstantScheduledThreadPoolExecutor();
+        ExecutorService executorService = new InstantExecutorService();
+        BackendActivator backendActivator = new BackendActivator(1, scheduledExecutorService, scheduledExecutorService,
+                                                                 scheduledExecutorService, executorService, countDownLatch);
+
+        backendActivator.activateBackend(backend, mock(Runnable.class));
+        backendActivator.deactivateBackend(backend, mock(Runnable.class));
+
+        // This should guarantee we overshoot the timeout if the implementation
+        // is broken (by not waiting on the latch, or not counting down).
+        backendActivator.waitForAllBackendsToDeactivate(60000);
+    }
+
+    @Test
+    public void testCannotAddActivationAfterShutdown() {
+        Backend backend = mock(Backend.class);
+        when(backend.activate()).thenReturn(true);
+        when(backend.deactivate()).thenReturn(true);
+        Runnable runnable = mock(Runnable.class);
+        BackendActivator backendActivator = new BackendActivator(2, 10);
+
+        backendActivator.shutdown();
+
+        assertEquals(0, backendActivator.getNumberOfBackendsActive());
+        backendActivator.activateBackend(backend, runnable);
+        assertEquals(0, backendActivator.getNumberOfBackendsActive());
+    }
+
+    @Test
+    public void testCannotAddDeactivationAfterShutdown() {
+        Backend backend = mock(Backend.class);
+        when(backend.activate()).thenReturn(true);
+        when(backend.deactivate()).thenReturn(true);
+        Runnable runnable = mock(Runnable.class);
+        CountDownLatch countDownLatch = new CountDownLatch(1);
+        ScheduledExecutorService scheduledExecutorService = new InstantScheduledThreadPoolExecutor();
+        ExecutorService executorService = new InstantExecutorService();
+        BackendActivator backendActivator = new BackendActivator(1, scheduledExecutorService, scheduledExecutorService,
+                                                                 scheduledExecutorService, executorService, countDownLatch);
+
+        backendActivator.activateBackend(backend, runnable);
+        assertEquals(1, backendActivator.getNumberOfBackendsActive());
+
+        backendActivator.shutdown();
+
+        backendActivator.deactivateBackend(backend, runnable);
+        assertEquals(1, backendActivator.getNumberOfBackendsActive());
+    }
+
+    private static class InstantScheduledThreadPoolExecutor extends ScheduledThreadPoolExecutor {
+        private InstantScheduledThreadPoolExecutor() {
+            super(1);
+        }
+
+        @Override
+        public ScheduledFuture<?> schedule(Runnable runnable, long timeout, TimeUnit timeoutTimeUnit) {
+            ScheduledFuture scheduledFutureMock = mock(ScheduledFuture.class);
+            when(scheduledFutureMock.isDone()).thenReturn(true);
+            runnable.run();
+            return scheduledFutureMock;
+        }
+    }
+
+    private static class InstantExecutorService extends ThreadPoolExecutor {
+        private InstantExecutorService() {
+            super(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue());
+        }
+
+        @Override
+        public Future<?> submit(Runnable runnable) {
+            Future futureMock = mock(Future.class);
+            when(futureMock.isDone()).thenReturn(true);
+            runnable.run();
+            return futureMock;
+        }
+    }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/agent/core/src/test/java/com/redhat/thermostat/backend/adverse/AdverseBackend.java	Wed Oct 04 15:08:14 2017 -0400
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2012-2017 Red Hat, Inc.
+ *
+ * This file is part of Thermostat.
+ *
+ * Thermostat is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; either version 2, or (at your
+ * option) any later version.
+ *
+ * Thermostat is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Thermostat; see the file COPYING.  If not see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Linking this code with other modules is making a combined work
+ * based on this code.  Thus, the terms and conditions of the GNU
+ * General Public License cover the whole combination.
+ *
+ * As a special exception, the copyright holders of this code give
+ * you permission to link this code with independent modules to
+ * produce an executable, regardless of the license terms of these
+ * independent modules, and to copy and distribute the resulting
+ * executable under terms of your choice, provided that you also
+ * meet, for each linked independent module, the terms and conditions
+ * of the license of that module.  An independent module is a module
+ * which is not derived from or based on this code.  If you modify
+ * this code, you may extend this exception to your version of the
+ * library, but you are not obligated to do so.  If you do not wish
+ * to do so, delete this exception statement from your version.
+ */
+
+package com.redhat.thermostat.backend.adverse;
+
+import com.redhat.thermostat.backend.BaseBackend;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Deactivate;
+
+import java.util.concurrent.CountDownLatch;
+
+public abstract class AdverseBackend extends BaseBackend {
+
+    protected final CountDownLatch latch;
+    protected volatile boolean wasInterrupted;
+
+    public AdverseBackend(String name, String description, String vendor) {
+        super(name, description, vendor);
+        this.latch = new CountDownLatch(1);
+    }
+
+    public void waitUntilInterrupted() {
+        while (!wasInterrupted) {
+        }
+    }
+
+    @Override
+    public boolean activate() {
+        return true;
+    }
+
+    @Override
+    public boolean deactivate() {
+        return true;
+    }
+
+    @Override
+    public boolean isActive() {
+        return false;
+    }
+
+    @Activate
+    public void activateDS() {
+        // Nothing. Make DS happy.
+    }
+
+    @Deactivate
+    public void deactivateDS() {
+        // Nothing. Make DS happy.
+    }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/agent/core/src/test/java/com/redhat/thermostat/backend/adverse/AdverseBackendActivation.java	Wed Oct 04 15:08:14 2017 -0400
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2012-2017 Red Hat, Inc.
+ *
+ * This file is part of Thermostat.
+ *
+ * Thermostat is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; either version 2, or (at your
+ * option) any later version.
+ *
+ * Thermostat is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Thermostat; see the file COPYING.  If not see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Linking this code with other modules is making a combined work
+ * based on this code.  Thus, the terms and conditions of the GNU
+ * General Public License cover the whole combination.
+ *
+ * As a special exception, the copyright holders of this code give
+ * you permission to link this code with independent modules to
+ * produce an executable, regardless of the license terms of these
+ * independent modules, and to copy and distribute the resulting
+ * executable under terms of your choice, provided that you also
+ * meet, for each linked independent module, the terms and conditions
+ * of the license of that module.  An independent module is a module
+ * which is not derived from or based on this code.  If you modify
+ * this code, you may extend this exception to your version of the
+ * library, but you are not obligated to do so.  If you do not wish
+ * to do so, delete this exception statement from your version.
+ */
+
+package com.redhat.thermostat.backend.adverse;
+
+import com.redhat.thermostat.common.utils.LoggingUtils;
+
+import java.util.logging.Logger;
+
+public class AdverseBackendActivation extends AdverseBackend {
+
+    private static final Logger logger = LoggingUtils.getLogger(AdverseBackendActivation.class);
+
+    public AdverseBackendActivation() {
+        super("Bad activation backend", "Starves other backends", "Adversary");
+    }
+
+    @Override
+    public boolean activate() {
+        try {
+            logger.info("I'm going to never return. Bye!");
+            latch.await(); // This never returns.
+        } catch (InterruptedException e) {
+            logger.info("Adverse activation backend latch interrupted");
+            wasInterrupted = true;
+        }
+        return true;
+    }
+
+    @Override
+    public synchronized boolean deactivate() {
+        return true;
+    }
+
+    @Override
+    public boolean isActive() {
+        return false;
+    }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/agent/core/src/test/java/com/redhat/thermostat/backend/adverse/AdverseBackendDeactivation.java	Wed Oct 04 15:08:14 2017 -0400
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2012-2017 Red Hat, Inc.
+ *
+ * This file is part of Thermostat.
+ *
+ * Thermostat is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; either version 2, or (at your
+ * option) any later version.
+ *
+ * Thermostat is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Thermostat; see the file COPYING.  If not see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Linking this code with other modules is making a combined work
+ * based on this code.  Thus, the terms and conditions of the GNU
+ * General Public License cover the whole combination.
+ *
+ * As a special exception, the copyright holders of this code give
+ * you permission to link this code with independent modules to
+ * produce an executable, regardless of the license terms of these
+ * independent modules, and to copy and distribute the resulting
+ * executable under terms of your choice, provided that you also
+ * meet, for each linked independent module, the terms and conditions
+ * of the license of that module.  An independent module is a module
+ * which is not derived from or based on this code.  If you modify
+ * this code, you may extend this exception to your version of the
+ * library, but you are not obligated to do so.  If you do not wish
+ * to do so, delete this exception statement from your version.
+ */
+
+package com.redhat.thermostat.backend.adverse;
+
+import com.redhat.thermostat.common.utils.LoggingUtils;
+
+import java.util.logging.Logger;
+
+public class AdverseBackendDeactivation extends AdverseBackend {
+
+    private static final Logger logger = LoggingUtils.getLogger(AdverseBackendDeactivation.class);
+    private boolean isActive;
+
+    public AdverseBackendDeactivation() {
+        super("Bad deactivating backend", "Starves other backends for deactivation", "Adversary Deactivation");
+    }
+
+    @Override
+    public boolean activate() {
+        isActive = true;
+        return true;
+    }
+
+    @Override
+    public boolean deactivate() {
+        try {
+            logger.info("I'm going to never return for deactivation. Bye!");
+            latch.await(); // This never returns.
+        } catch (InterruptedException e) {
+            logger.info("Adverse backend deactivation latch interrupted");
+            wasInterrupted = true;
+        }
+        isActive = false;
+        return true;
+    }
+
+    @Override
+    public boolean isActive() {
+        return isActive;
+    }
+}