-
Type:
Bug
-
Status: Resolved (View Workflow)
-
Priority:
Blocker
-
Resolution: Done
-
Affects Version/s: None
-
Fix Version/s: 2.0.0
-
Component/s: None
-
Labels:None
-
Environment:
CORD single node pod
-
Story Points:3
-
Sprint:CORD Sprint 8, Dangerous Addition Sprint #1
-
Epic Link:
@hyunsun: something failed in one of my ONOS containers last night. This experiment had been up less than 24 hours. Looks like there is plenty of free RAM.
2016-11-11 16:21:10,048 | WARN | qtp1781550494-47 | ServerErrorMapper | 145 - org.onosproject.onlab-rest - 1.8.0.SNAPSHOT | Unhandled REST exception org.onosproject.store.service.ConsistentMapException$Timeout: cordvtn-vtnportstore at org.onosproject.store.primitives.DefaultConsistentMap.complete(DefaultConsistentMap.java:228)[74:org.onosproject.onos-api:1.8.0.SNAPSHOT] at org.onosproject.store.primitives.DefaultConsistentMap.remove(DefaultConsistentMap.java:128)[74:org.onosproject.onos-api:1.8.0.SNAPSHOT] at org.opencord.cordvtn.impl.DistributedCordVtnStore.removeVtnPort(DistributedCordVtnStore.java:273)[170:org.opencord.vtn:1.1.0.SNAPSHOT] at org.opencord.cordvtn.impl.CordVtnManager.removeServicePort(CordVtnManager.java:232)[170:org.opencord.vtn:1.1.0.SNAPSHOT] at org.opencord.cordvtn.rest.ServicePortWebResource.deleteServicePort(ServicePortWebResource.java:188)[170:org.opencord.vtn:1.1.0.SNAPSHOT] at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)[:1.8.0_111] at java.lang.reflect.Method.invoke(Method.java:498)[:1.8.0_111] at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:160)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.internal.Errors.process(Errors.java:315)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.internal.Errors.process(Errors.java:297)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.internal.Errors.process(Errors.java:267)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:473)[142:org.glassfish.jersey.containers.jersey-container-servlet-core:2.22.2] at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:427)[142:org.glassfish.jersey.containers.jersey-container-servlet-core:2.22.2] at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:388)[142:org.glassfish.jersey.containers.jersey-container-servlet-core:2.22.2] at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:341)[142:org.glassfish.jersey.containers.jersey-container-servlet-core:2.22.2] at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:228)[142:org.glassfish.jersey.containers.jersey-container-servlet-core:2.22.2] at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:503)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.ops4j.pax.web.service.jetty.internal.HttpServiceServletHandler.doHandle(HttpServiceServletHandler.java:69)[111:org.ops4j.pax.web.pax-web-jetty:3.2.6] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:522)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.ops4j.pax.web.service.jetty.internal.HttpServiceContext.doHandle(HttpServiceContext.java:240)[111:org.ops4j.pax.web.pax-web-jetty:3.2.6] at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.ops4j.pax.web.service.jetty.internal.JettyServerHandlerCollection.handle(JettyServerHandlerCollection.java:75)[111:org.ops4j.pax.web.pax-web-jetty:3.2.6] at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.Server.handle(Server.java:370)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)[102:org.eclipse.jetty.aggregate.jetty-all-server:8.1.17.v20150415] at java.lang.Thread.run(Thread.java:745)[:1.8.0_111] 2016-11-11 16:21:11,584 | INFO | f]-nio2-thread-1 | ServerSession | 30 - org.apache.sshd.core - 0.14.0 | Server session created from /10.1.0.1:42778 2016-11-11 16:21:11,594 | INFO | f]-nio2-thread-1 | SimpleGeneratorHostKeyProvider | 30 - org.apache.sshd.core - 0.14.0 | Generating host key... 2016-11-11 16:21:12,325 | WARN | qtp1781550494-48 | ServerErrorMapper | 145 - org.onosproject.onlab-rest - 1.8.0.SNAPSHOT | Unhandled REST exception org.onosproject.store.service.ConsistentMapException$Timeout: onos-apps at org.onosproject.store.primitives.DefaultConsistentMap.complete(DefaultConsistentMap.java:228) at org.onosproject.store.primitives.DefaultConsistentMap.get(DefaultConsistentMap.java:85) at org.onosproject.store.app.DistributedApplicationStore.getApplication(DistributedApplicationStore.java:287) at org.onosproject.app.impl.ApplicationManager.getApplication(ApplicationManager.java:121) at org.onosproject.rest.resources.ApplicationsWebResource.response(ApplicationsWebResource.java:246)[146:org.onosproject.onos-rest:1.8.0.SNAPSHOT] at org.onosproject.rest.resources.ApplicationsWebResource.activateApp(ApplicationsWebResource.java:170)[146:org.onosproject.onos-rest:1.8.0.SNAPSHOT] at sun.reflect.GeneratedMethodAccessor23.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)[:1.8.0_111] at java.lang.reflect.Method.invoke(Method.java:498)[:1.8.0_111] at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:160)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326)[136:org.glassfish.jersey.core.jersey-server:2.22.2] at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.internal.Errors.process(Errors.java:315)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.internal.Errors.process(Errors.java:297)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.internal.Errors.process(Errors.java:267)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317)[128:org.glassfish.jersey.core.jersey-common:2.22.2] at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305)[136:org.glassfish.jersey.core.jersey...
make cord-monitoringservice is successful,but the vm's are in error state.
onos> cordvtn-nodes
Error executing command: cordvtn-nodestore
Here's the first timeout from the log collected from the Scott's environment.
2016-11-11 08:08:37,989 | INFO | -message-handler | CordVtnPipeline | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:37,991 | INFO | -message-handler | CodecRegistrator | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,024 | INFO | -message-handler | DistributedCordVtnStore | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,043 | INFO | -message-handler | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,046 | INFO | -message-handler | InstanceManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,063 | INFO | -message-handler | CordVtnNodeManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,065 | INFO | -message-handler | CordVtnArpProxy | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,075 | INFO | -message-handler | DefaultInstanceHandler | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,076 | INFO | -message-handler | AccessAgentInstanceHandler | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,077 | INFO | -message-handler | ManagementInstanceHandler | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,079 | INFO | -message-handler | VsgInstanceHandler | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,087 | INFO | -message-handler | DependencyManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Started 2016-11-11 08:08:38,419 | INFO | -message-handler | HttpServiceFactoryImpl | 110 - org.ops4j.pax.web.pax-web-runtime - 3.2.6 | Binding bundle: [org.opencord.vtn [170]] to http service 2016-11-11 08:08:38,430 | INFO | -message-handler | ApplicationManager | 76 - org.onosproject.onos-core-net - 1.8.0.SNAPSHOT | Application org.opencord.vtn has been activated 2016-11-11 08:08:39,192 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Installing feature onos-optical-model 1.8.0-SNAPSHOT 2016-11-11 08:08:39,192 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Found installed feature onos-api 1.8.0-SNAPSHOT 2016-11-11 08:08:39,196 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Found installed feature scr-condition-webconsole_0_0_0 3.0.5 2016-11-11 08:08:39,197 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Found installed feature standard-condition-webconsole_0_0_0 3.0.5 2016-11-11 08:08:39,197 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Found installed feature webconsole-condition-scr_0_0_0 3.0.5 2016-11-11 08:08:39,238 | INFO | -message-handler | OpticalModelLoader | 172 - org.onosproject.onos-apps-optical-model - 1.8.0.SNAPSHOT | Started 2016-11-11 08:08:39,239 | INFO | -message-handler | ApplicationManager | 76 - org.onosproject.onos-core-net - 1.8.0.SNAPSHOT | Application org.onosproject.optical-model has been activated 2016-11-11 08:08:39,241 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Installing feature onos-providers-openflow-base 1.8.0-SNAPSHOT 2016-11-11 08:08:39,241 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Found installed feature onos-api 1.8.0-SNAPSHOT 2016-11-11 08:08:39,254 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Found installed feature scr-condition-webconsole_0_0_0 3.0.5 2016-11-11 08:08:39,254 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Found installed feature standard-condition-webconsole_0_0_0 3.0.5 2016-11-11 08:08:39,255 | INFO | -message-handler | FeaturesServiceImpl | 20 - org.apache.karaf.features.core - 3.0.5 | Found installed feature webconsole-condition-scr_0_0_0 3.0.5 2016-11-11 08:08:39,322 | INFO | -message-handler | Controller | 173 - org.onosproject.onos-protocols-openflow-ctl - 1.8.0.SNAPSHOT | Starting OpenFlow IO 2016-11-11 08:08:39,322 | INFO | -message-handler | Controller | 173 - org.onosproject.onos-protocols-openflow-ctl - 1.8.0.SNAPSHOT | OpenFlow Security is disabled 2016-11-11 08:08:39,374 | INFO | -message-handler | Controller | 173 - org.onosproject.onos-protocols-openflow-ctl - 1.8.0.SNAPSHOT | Listening for switch connections on 0.0.0.0/0.0.0.0:6633 2016-11-11 08:08:39,379 | INFO | -message-handler | Controller | 173 - org.onosproject.onos-protocols-openflow-ctl - 1.8.0.SNAPSHOT | Listening for switch connections on 0.0.0.0/0.0.0.0:6653 2016-11-11 08:08:39,390 | INFO | -message-handler | OpenFlowDeviceProvider | 174 - org.onosproject.onos-providers-openflow-device - 1.8.0.SNAPSHOT | Settings: portStatsPollFrequency=5 2016-11-11 08:08:39,390 | INFO | -message-handler | OpenFlowDeviceProvider | 174 - org.onosproject.onos-providers-openflow-device - 1.8.0.SNAPSHOT | Started 2016-11-11 08:08:39,395 | INFO | -message-handler | OpenFlowPacketProvider | 175 - org.onosproject.onos-providers-openflow-packet - 1.8.0.SNAPSHOT | Started 2016-11-11 08:08:39,403 | INFO | -message-handler | OpenFlowRuleProvider | 176 - org.onosproject.onos-providers-openflow-flow - 1.8.0.SNAPSHOT | Settings: flowPollFrequency=5 2016-11-11 08:08:39,403 | INFO | -message-handler | OpenFlowRuleProvider | 176 - org.onosproject.onos-providers-openflow-flow - 1.8.0.SNAPSHOT | Settings: adaptiveFlowSampling=false 2016-11-11 08:08:39,411 | INFO | -message-handler | OpenFlowRuleProvider | 176 - org.onosproject.onos-providers-openflow-flow - 1.8.0.SNAPSHOT | Started with flowPollFrequency = 5, adaptiveFlowSampling = false 2016-11-11 08:08:39,418 | INFO | -message-handler | OpenFlowGroupProvider | 177 - org.onosproject.onos-providers-openflow-group - 1.8.0.SNAPSHOT | Started 2016-11-11 08:08:39,425 | INFO | -message-handler | ApplicationManager | 76 - org.onosproject.onos-core-net - 1.8.0.SNAPSHOT | Application org.onosproject.openflow-base has been activated 2016-11-11 08:30:53,010 | INFO | qtp1781550494-48 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Network created f0a4b391-dd47-42d2-b93b-675d1ebfc3f2 2016-11-11 08:30:54,225 | INFO | qtp1781550494-46 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Subnet created b8e992ba-86f7-4a6c-9f19-338c555b1fb4 2016-11-11 08:30:58,278 | INFO | qtp1781550494-47 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Network created c693f41b-1fdb-49e1-a711-0d1abafd58fc 2016-11-11 08:30:59,377 | INFO | qtp1781550494-46 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Subnet created 018cb334-6ed4-4bde-a48c-80a803854083 2016-11-11 08:30:59,616 | INFO | qtp1781550494-47 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | VTN network created f0a4b391-dd47-42d2-b93b-675d1ebfc3f2 2016-11-11 08:31:01,607 | INFO | qtp1781550494-48 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Port created a63b147c-8ca7-45a1-9710-eab256749905 2016-11-11 08:31:04,926 | INFO | qtp1781550494-45 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | VTN network created c693f41b-1fdb-49e1-a711-0d1abafd58fc 2016-11-11 08:31:05,124 | INFO | qtp1781550494-45 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | VTN port created a63b147c-8ca7-45a1-9710-eab256749905 2016-11-11 08:31:14,511 | INFO | qtp1781550494-46 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Port updated a63b147c-8ca7-45a1-9710-eab256749905 2016-11-11 08:31:14,591 | INFO | qtp1781550494-45 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Port updated a63b147c-8ca7-45a1-9710-eab256749905 2016-11-11 08:31:14,744 | INFO | qtp1781550494-46 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Port created c65e190e-a9a9-4475-9a82-796fd48a0a52 2016-11-11 08:31:14,822 | INFO | qtp1781550494-48 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Port updated c65e190e-a9a9-4475-9a82-796fd48a0a52 2016-11-11 08:31:37,115 | INFO | qtp1781550494-47 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Port updated a63b147c-8ca7-45a1-9710-eab256749905 2016-11-11 08:31:37,236 | INFO | qtp1781550494-46 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | VTN port removed c65e190e-a9a9-4475-9a82-796fd48a0a52 2016-11-11 08:31:37,237 | INFO | qtp1781550494-46 | CordVtnManager | 170 - org.opencord.vtn - 1.1.0.SNAPSHOT | Port removed c65e190e-a9a9-4475-9a82-796fd48a0a52 2016-11-11 09:42:21,579 | INFO | 9876-partition-1 | ServerStateMachine | 40 - io.atomix.all - 1.0.0.rc9 | /10.1.0.13:9876 - Taking snapshot 6827 2016-11-11 09:42:24,358 | INFO | 9876-partition-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 10:02:34,172 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MAJOR 2016-11-11 11:02:34,167 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MAJOR 2016-11-11 11:18:56,284 | INFO | 9876-partition-1 | ServerStateMachine | 40 - io.atomix.all - 1.0.0.rc9 | /10.1.0.13:9876 - Taking snapshot 13288 2016-11-11 11:18:59,092 | INFO | 9876-partition-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:19:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:20:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:21:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:22:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:23:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:24:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:25:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:26:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:27:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:28:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:29:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:30:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:31:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:32:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:33:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:34:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:35:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:36:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:37:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:38:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:39:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:40:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:41:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:42:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:43:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:44:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:45:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:46:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:47:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:48:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:49:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:50:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:51:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:52:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:53:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:54:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:55:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:56:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:57:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:58:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 11:59:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 12:00:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 12:01:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 12:02:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 12:02:34,351 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MAJOR 2016-11-11 12:55:25,917 | INFO | 9876-partition-1 | ServerStateMachine | 40 - io.atomix.all - 1.0.0.rc9 | /10.1.0.13:9876 - Taking snapshot 19740 2016-11-11 12:55:26,362 | INFO | 9876-partition-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 12:55:34,166 | INFO | ycat-compactor-1 | Compactor | 40 - io.atomix.all - 1.0.0.rc9 | Compacting log with compaction: MINOR 2016-11-11 12:55:38,192 | WARN | qtp1781550494-45 | ServerErrorMapper | 145 - org.onosproject.onlab-rest - 1.8.0.SNAPSHOT | Unhandled REST exception org.onosproject.store.service.ConsistentMapException$Timeout: onos-apps at org.onosproject.store.primitives.DefaultConsistentMap.complete(DefaultConsistentMap.java:228) at org.onosproject.store.primitives.DefaultConsistentMap.get(DefaultConsistentMap.java:85) at org.onosproject.store.app.DistributedApplicationStore.getApplication(DistributedApplicationStore.java:287) at org.onosproject.app.impl.ApplicationManager.getApplication(ApplicationManager.java:121) at org.onosproject.rest.resources.ApplicationsWebResource.response(ApplicationsWebResource.java:246)[146:org.onosproject.onos-rest:1.8.0.SNAPSHOT] at org.onosproject.rest.resources.ApplicationsWebResource.activateApp(ApplicationsWebResource.java:170)[146:org.onosproject.onos-rest:1.8.0.SNAPSHOT] at sun.reflect.GeneratedMethodAccessor23.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)[:1.8.0_111] at java.lang.reflect.Method.invoke(Method.java:498)[:1.8.0_111]