(HttpClient.java:211)
at sun.net.www.http.HttpClient.New(HttpClient.java:308)
at sun.net.www.http.HttpClient.New(HttpClient.java:326)
at sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient(HttpURLConnection.java:996)
at sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:932)
at sun.net.www.protocol.http.HttpURLConnection.connect(HttpURLConnection.java:850)
at sun.net.www.protocol.http.HttpURLConnection.getOutputStream(HttpURLConnection.java:1091)
at org.jboss.remoting.transport.http.HTTPClientInvoker.getOutputStream(HTTPClientInvoker.java:1301)
at org.jboss.remoting.transport.http.HTTPClientInvoker.useHttpURLConnection(HTTPClientInvoker.java:367)
at org.jboss.remoting.transport.http.HTTPClientInvoker.makeInvocation(HTTPClientInvoker.java:253)
... 22 more
2014-11-05 06:51:50,975 ERROR [ClientCommandSenderTask Thread #2] (enterprise.communications.command.client.ClientCommandSenderTask)- {ClientCommandSenderTask.send-failed}Failed to send command [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.agent-name=fbr-ha.bc.jonqe.lab.eng.bos.redhat.com, rhq.externalizable-strategy=AGENT, rhq.security-token=droMLlWh5RZBojloyDCrX3JxMre8hyoN1bMLc2d9yezzpQE2dJ1amImap1LKlzswVcA=, rhq.retry=1, rhq.guaranteed-delivery=true, rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeMeasurementReport], targetInterfaceName=org.rhq.core.clientapi.server.measurement.MeasurementServerService}]]. Cause: java.lang.InterruptedException:null. Cause: java.lang.InterruptedException
2014-11-05 06:51:50,976 WARN [ClientCommandSenderTask Thread #2] (enterprise.communications.command.client.ClientCommandSenderTask)- {ClientCommandSenderTask.queuing-failed-command}The command that failed has its guaranteed-delivery flag set so it is being queued again
2014-11-05 06:51:50,984 WARN [RHQ Server Polling Thread] (enterprise.communications.command.client.ServerPollingThread)- {ServerPollingThread.server-offline}The server has gone offline; client has been told to stop sending commands
2014-11-05 06:51:55,231 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=13660, uuid=ab45a606-e3a0-48d8-a43c-5b931a2e1946, type={JBossAS7}Singleton Bean Runtime, key=deployment=rhq.ear,subdeployment=rhq-server.jar,subsystem=ejb3,singleton-bean=CustomJaasDeploymentService, name=CustomJaasDeploymentService, parent=ejb3] - cause: java.lang.NullPointerException:null
2014-11-05 06:51:57,238 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=14127, uuid=ee293ad8-58ed-4910-80ca-2a023ddaa93e, type={JBossAS7}Stateless Session Bean Runtime, key=deployment=rhq.ear,subdeployment=rhq-server.jar,subsystem=ejb3,stateless-session-bean=MeasurementChartsManagerBean, name=MeasurementChartsManagerBean, parent=ejb3] - cause: java.lang.NullPointerException:null
2014-11-05 06:51:57,240 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=14137, uuid=4a1a2ea3-2b51-48c6-bb24-1d2d763d5b0c, type={JBossAS7}Stateless Session Bean Runtime, key=deployment=rhq.ear,subdeployment=rhq-server.jar,subsystem=ejb3,stateless-session-bean=ServerPluginManagerBean, name=ServerPluginManagerBean, parent=ejb3] - cause: java.lang.NullPointerException:null
2014-11-05 06:51:59,232 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=13974, uuid=2196c377-b8dd-4564-a6a8-f73950ef3ca1, type={JBossAS7}Stateless Session Bean Runtime, key=deployment=rhq.ear,subdeployment=rhq-server.jar,subsystem=ejb3,stateless-session-bean=PluginConfigurationMetadataManagerBean, name=PluginConfigurationMetadataManagerBean, parent=ejb3] - cause: java.lang.NullPointerException:null
2014-11-05 06:52:09,231 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=13949, uuid=5ed90449-0b2a-461a-b3e1-1acdf3044cd4, type={JBossAS7}Singleton Bean Runtime, key=deployment=rhq.ear,subdeployment=rhq-server.jar,subsystem=ejb3,singleton-bean=PluginDeploymentScanner, name=PluginDeploymentScanner, parent=ejb3] - cause: java.lang.NullPointerException:null
2014-11-05 06:52:17,526 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:52:17,527 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:52:47,601 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:52:47,601 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:52:51,003 INFO [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 06:52:51,006 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)
2014-11-05 06:52:51,007 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)]
2014-11-05 06:52:51,008 ERROR [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.CannotConnectException:Can not connect http client invoker after 1 attempt(s) -> java.net.ConnectException:Connection refused. Cause: org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)
2014-11-05 06:53:17,684 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:53:17,685 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:53:47,226 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 06:53:47,382 INFO [ResourceDiscoveryComponent.invoker.daemon-4] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 06:53:47,537 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 06:53:47,537 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 06:53:47,537 WARN [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Exception caught while executing server discovery scan.
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.timeout=0, rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeInventoryReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeInventoryReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1265)
at org.rhq.core.pc.inventory.AutoDiscoveryExecutor.call(AutoDiscoveryExecutor.java:121)
at org.rhq.core.pc.inventory.AutoDiscoveryExecutor.run(AutoDiscoveryExecutor.java:83)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:53:47,747 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:53:47,748 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:53:51,020 INFO [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 06:53:51,023 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)
2014-11-05 06:53:51,024 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)]
2014-11-05 06:53:51,024 ERROR [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.CannotConnectException:Can not connect http client invoker after 1 attempt(s) -> java.net.ConnectException:Connection refused. Cause: org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)
2014-11-05 06:54:17,829 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:54:17,829 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:54:47,886 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:54:47,887 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:54:51,037 INFO [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 06:54:51,039 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)
2014-11-05 06:54:51,040 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)]
2014-11-05 06:54:51,041 ERROR [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.CannotConnectException:Can not connect http client invoker after 1 attempt(s) -> java.net.ConnectException:Connection refused. Cause: org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)
2014-11-05 06:55:17,975 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:55:17,975 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:55:48,009 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:55:48,009 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:55:51,053 INFO [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 06:55:51,056 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)
2014-11-05 06:55:51,057 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)]
2014-11-05 06:55:51,057 ERROR [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.CannotConnectException:Can not connect http client invoker after 1 attempt(s) -> java.net.ConnectException:Connection refused. Cause: org.jboss.remoting.CannotConnectException: Can not connect http client invoker after 1 attempt(s)
2014-11-05 06:56:18,087 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:56:18,087 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:56:48,131 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:56:48,132 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:56:51,335 INFO [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 06:56:51,338 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 06:56:51,340 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
]
2014-11-05 06:56:51,340 ERROR [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.transport.http.WebServerError:JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 06:57:45,813 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:57:45,814 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:57:51,380 INFO [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 06:57:51,403 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 06:57:51,406 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
]
2014-11-05 06:57:51,406 ERROR [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.transport.http.WebServerError:JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 06:58:18,034 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:58:18,035 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:58:49,238 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:58:49,239 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 06:58:51,780 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.server-thinks-agent-is-down}The server thinks this agent is down. Will notify the server of up-to-date information when possible.
2014-11-05 06:58:51,928 INFO [RHQ Server Polling Thread] (enterprise.communications.command.client.ServerPollingThread)- {ServerPollingThread.server-online}The server has come back online; client has been told to start sending commands again
2014-11-05 06:59:06,499 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 06:59:20,937 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 06:59:37,033 INFO [RHQ Agent Ping Thread-1] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.ping-executor.stop-polling-resume-ping}Stopping polling and resuming pinging
2014-11-05 07:00:00,631 INFO [ResourceContainer.invoker.availCheck.daemon-21] (org.mc4j.ems.connection.ConnectionFactory)- Discovered libraries in 0 ms
2014-11-05 07:00:00,632 INFO [ResourceContainer.invoker.availCheck.daemon-21] (org.rhq.plugins.server.JBossAS7JMXComponent)- Loading AS7 connection [service:jmx:remoting-jmx://127.0.0.1:6999] with install path [/home/hudson/jon-server-3.3.0.CR01/jbossas/bin/client]...
2014-11-05 07:08:47,539 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 07:08:47,815 INFO [ResourceDiscoveryComponent.invoker.daemon-5] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 07:08:47,900 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 07:08:47,900 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 07:08:48,059 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 07:08:48,079 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 07:08:48,080 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 07:08:48,173 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 07:08:48,179 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 07:08:49,007 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 07:08:49,008 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 07:08:49,827 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 07:08:49,828 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 07:08:50,684 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 07:08:50,686 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 07:08:51,491 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 07:10:28,746 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 07:19:06,297 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 07:19:11,806 INFO [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- [PERF] Collection of measurements for [Resource[id=13703, uuid=bf47f049-0451-4ac9-a74f-5371a5f7d351, type={JBossAS7}Hibernate Persistence Unit, key=deployment=rhq.ear,subdeployment=rhq-core-domain-ejb3.jar,subsystem=jpa,hibernate-persistence-unit=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, name=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, parent=jpa]] (component=[org.rhq.modules.plugins.jbossas7.HibernatePersistenceUnitComponent@50b0ab76]) took [4626]ms
2014-11-05 07:23:51,491 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 07:23:51,769 INFO [ResourceDiscoveryComponent.invoker.daemon-7] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 07:23:51,859 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 07:23:51,859 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 07:23:51,968 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 07:23:51,968 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 07:23:51,968 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 07:23:52,051 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 07:23:52,057 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 07:23:52,879 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 07:23:52,880 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 07:23:53,697 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 07:23:53,698 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 07:23:54,535 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 07:23:54,537 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 07:23:55,340 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 07:38:55,340 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 07:38:55,597 INFO [ResourceDiscoveryComponent.invoker.daemon-10] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 07:38:55,676 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 07:38:55,677 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 07:38:55,789 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 07:38:55,789 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 07:38:55,790 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 07:38:55,850 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 07:38:55,856 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 07:38:56,677 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 07:38:56,679 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 07:38:57,500 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 07:38:57,500 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 07:38:58,329 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 07:38:58,331 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 07:38:59,136 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 07:39:06,259 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 07:39:11,740 INFO [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- [PERF] Collection of measurements for [Resource[id=13703, uuid=bf47f049-0451-4ac9-a74f-5371a5f7d351, type={JBossAS7}Hibernate Persistence Unit, key=deployment=rhq.ear,subdeployment=rhq-core-domain-ejb3.jar,subsystem=jpa,hibernate-persistence-unit=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, name=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, parent=jpa]] (component=[org.rhq.modules.plugins.jbossas7.HibernatePersistenceUnitComponent@50b0ab76]) took [4581]ms
2014-11-05 07:45:36,183 INFO [ResourceContainer.invoker.daemon-12] (org.rhq.plugins.jmx.MBeanResourceComponent)- Configuration management is currently disabled for the ResourceType[id=0, name=java.util.logging, plugin=RHQAgent, category=Service] Resource with key [java.util.logging:type=Logging] - it can be enabled in the Resource's Inventory > Connection tab.
2014-11-05 07:45:36,238 INFO [ResourceContainer.invoker.daemon-12] (org.rhq.plugins.jmx.MBeanResourceComponent)- Configuration management is currently disabled for the ResourceType[id=0, name=java.util.logging, plugin=RHQStorage, category=Service] Resource with key [java.util.logging:type=Logging] - it can be enabled in the Resource's Inventory > Connection tab.
2014-11-05 07:46:42,237 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=13324, uuid=0a76d3b7-3b92-4007-b7b2-b3ef3477e5b2, type={RHQServer}RHQ Server Storage Client Subsystem, key=rhq:service=StorageClientManager, name=Storage Client Subsystem, parent=RHQ Server Subsystems] - cause: java.lang.RuntimeException:Unable to load attributes on bean [rhq:service=StorageClientManager] null -> java.lang.reflect.UndeclaredThrowableException:null -> org.jboss.remoting3.NotOpenException:Writes closed
2014-11-05 07:46:53,235 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=13324, uuid=0a76d3b7-3b92-4007-b7b2-b3ef3477e5b2, type={RHQServer}RHQ Server Storage Client Subsystem, key=rhq:service=StorageClientManager, name=Storage Client Subsystem, parent=RHQ Server Subsystems] - cause: java.lang.RuntimeException:Unable to load attributes on bean [rhq:service=StorageClientManager] null -> java.lang.reflect.UndeclaredThrowableException:null -> org.jboss.remoting3.NotOpenException:Writes closed
2014-11-05 07:47:01,332 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 07:47:01,366 INFO [ClientCommandSenderTask Timer Thread #22] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 07:47:01,369 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:01,376 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
]
2014-11-05 07:47:01,376 ERROR [InventoryManager.availability-1] (enterprise.communications.command.client.ClientCommandSenderTask)- {ClientCommandSenderTask.send-failed}Failed to send command [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.agent-name=fbr-ha.bc.jonqe.lab.eng.bos.redhat.com, rhq.externalizable-strategy=AGENT, rhq.security-token=droMLlWh5RZBojloyDCrX3JxMre8hyoN1bMLc2d9yezzpQE2dJ1amImap1LKlzswVcA=, rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]. Cause: org.jboss.remoting.transport.http.WebServerError:JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:01,377 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.reflect.UndeclaredThrowableException
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
at org.jboss.remoting.transport.http.HTTPClientInvoker.useHttpURLConnection(HTTPClientInvoker.java:497)
at org.jboss.remoting.transport.http.HTTPClientInvoker.makeInvocation(HTTPClientInvoker.java:253)
at org.jboss.remoting.transport.http.HTTPClientInvoker.transport(HTTPClientInvoker.java:176)
at org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:169)
at org.jboss.remoting.Client.invoke(Client.java:2084)
at org.jboss.remoting.Client.invoke(Client.java:879)
at org.jboss.remoting.Client.invoke(Client.java:867)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.invoke(JBossRemotingRemoteCommunicator.java:501)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.rawSend(JBossRemotingRemoteCommunicator.java:346)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.sendWithoutCallbacks(JBossRemotingRemoteCommunicator.java:288)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.sendWithoutInitializeCallback(JBossRemotingRemoteCommunicator.java:307)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.send(JBossRemotingRemoteCommunicator.java:328)
at org.rhq.enterprise.communications.command.client.AbstractCommandClient.invoke(AbstractCommandClient.java:143)
at org.rhq.enterprise.communications.command.client.ClientCommandSender.send(ClientCommandSender.java:1084)
at org.rhq.enterprise.communications.command.client.ClientCommandSenderTask.send(ClientCommandSenderTask.java:229)
at org.rhq.enterprise.communications.command.client.ClientCommandSenderTask.call(ClientCommandSenderTask.java:107)
at org.rhq.enterprise.communications.command.client.ClientCommandSenderTask.call(ClientCommandSenderTask.java:55)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
... 3 more
2014-11-05 07:47:05,951 INFO [ClientCommandSenderTask Timer Thread #22] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 07:47:05,957 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:05,966 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
]
2014-11-05 07:47:05,966 ERROR [ClientCommandSenderTask Timer Thread #22] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.transport.http.WebServerError:JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 404 - /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message /jboss-remoting-servlet-invoker/ServerInvokerServlet
JBWEB000069: description JBWEB000124: The requested resource is not available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:31,433 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 07:47:31,451 INFO [ClientCommandSenderTask Timer Thread #22] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 07:47:31,456 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:31,460 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
]
2014-11-05 07:47:31,461 ERROR [ClientCommandSenderTask Timer Thread #22] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.transport.http.WebServerError:JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:31,461 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.reflect.UndeclaredThrowableException
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
at org.jboss.remoting.transport.http.HTTPClientInvoker.useHttpURLConnection(HTTPClientInvoker.java:497)
at org.jboss.remoting.transport.http.HTTPClientInvoker.makeInvocation(HTTPClientInvoker.java:253)
at org.jboss.remoting.transport.http.HTTPClientInvoker.transport(HTTPClientInvoker.java:176)
at org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:169)
at org.jboss.remoting.Client.invoke(Client.java:2084)
at org.jboss.remoting.Client.invoke(Client.java:879)
at org.jboss.remoting.Client.invoke(Client.java:867)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.invoke(JBossRemotingRemoteCommunicator.java:501)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.rawSend(JBossRemotingRemoteCommunicator.java:346)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.sendWithoutCallbacks(JBossRemotingRemoteCommunicator.java:288)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.sendWithoutInitializeCallback(JBossRemotingRemoteCommunicator.java:307)
at org.rhq.enterprise.agent.AgentMain.sendConnectRequestToServer(AgentMain.java:2269)
at org.rhq.enterprise.agent.ConnectAgentInitializeCallback.sendingInitialCommand(ConnectAgentInitializeCallback.java:43)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.invokeInitializeCallbackIfNeeded(JBossRemotingRemoteCommunicator.java:411)
at org.rhq.enterprise.communications.command.client.JBossRemotingRemoteCommunicator.send(JBossRemotingRemoteCommunicator.java:323)
at org.rhq.enterprise.communications.command.client.AbstractCommandClient.invoke(AbstractCommandClient.java:143)
at org.rhq.enterprise.communications.command.client.ClientCommandSender.send(ClientCommandSender.java:1084)
at org.rhq.enterprise.communications.command.client.ClientCommandSenderTask.send(ClientCommandSenderTask.java:229)
at org.rhq.enterprise.communications.command.client.ClientCommandSenderTask.call(ClientCommandSenderTask.java:107)
at org.rhq.enterprise.communications.command.client.ClientCommandSenderTask.call(ClientCommandSenderTask.java:55)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
... 3 more
2014-11-05 07:47:35,953 INFO [ClientCommandSenderTask Timer Thread #22] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 07:47:35,956 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:35,958 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
]
2014-11-05 07:47:35,958 ERROR [ClientCommandSenderTask Timer Thread #22] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.transport.http.WebServerError:JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:37,689 INFO [ClientCommandSenderTask Timer Thread #22] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 07:47:37,695 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:37,706 WARN [ClientCommandSenderTask Timer Thread #22] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
]
2014-11-05 07:47:37,707 ERROR [ClientCommandSenderTask Timer Thread #22] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.transport.http.WebServerError:JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:37,708 INFO [RHQ Agent Ping Thread-1] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.ping-executor.start-polling-after-exception}Starting polling to determine sender status (server ping failed)
2014-11-05 07:47:37,723 INFO [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://10.16.23.104:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
2014-11-05 07:47:37,728 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:37,732 WARN [RHQ Server Polling Thread] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
]
2014-11-05 07:47:37,733 ERROR [RHQ Server Polling Thread] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.jboss.remoting.transport.http.WebServerError:JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
. Cause: org.jboss.remoting.transport.http.WebServerError: JBoss Web/7.4.8.Final-redhat-4 - JBWEB000064: Error report JBWEB000065: HTTP Status 503 - Server is not ready yet
JBWEB000309: type JBWEB000067: Status report
JBWEB000068: message Server is not ready yet
JBWEB000069: description JBWEB000148: The requested service is not currently available.
JBoss Web/7.4.8.Final-redhat-4
2014-11-05 07:47:37,735 WARN [RHQ Server Polling Thread] (enterprise.communications.command.client.ServerPollingThread)- {ServerPollingThread.server-offline}The server has gone offline; client has been told to stop sending commands
2014-11-05 07:48:28,813 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 07:48:28,813 WARN [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
at com.sun.proxy.$Proxy4.mergeAvailabilityReport(Unknown Source)
at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1223)
at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 07:48:38,174 INFO [RHQ Server Polling Thread] (enterprise.communications.command.client.ServerPollingThread)- {ServerPollingThread.server-online}The server has come back online; client has been told to start sending commands again
2014-11-05 07:49:02,156 INFO [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
2014-11-05 07:49:37,710 INFO [RHQ Agent Ping Thread-1] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.ping-executor.stop-polling-resume-ping}Stopping polling and resuming pinging
2014-11-05 07:53:43,004 WARN [ConfigurationManager.threadpool-1] (rhq.core.pc.configuration.ConfigurationCheckExecutor)- An error occurred while checking for an updated Resource configuration for Resource[id=13012, uuid=09c208af-0fd6-4567-a490-8986bfebccb0, type={Postgres}Postgres Server, key=jdbc:postgresql://127.0.0.1:5432/postgres, name=rhq, parent=fbr-ha.bc.jonqe.lab.eng.bos.redhat.com, version=8.4.20].
java.io.IOException: PostgreSQL configuration file [/var/lib/pgsql/data/postgresql.conf] does not exist or is not readable. Make sure the user the RHQ Agent is running as has read permissions on the file and its parent directory.
at org.rhq.plugins.postgres.util.PostgresqlConfFile.(PostgresqlConfFile.java:64)
at org.rhq.plugins.postgres.PostgresServerComponent.getConfigurationFile(PostgresServerComponent.java:219)
at org.rhq.plugins.postgres.PostgresServerComponent.loadResourceConfiguration(PostgresServerComponent.java:238)
at sun.reflect.GeneratedMethodAccessor78.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.rhq.core.pc.inventory.ResourceContainer$ComponentInvocation.call(ResourceContainer.java:759)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 07:53:59,136 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 07:53:59,401 INFO [ResourceDiscoveryComponent.invoker.daemon-11] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 07:53:59,466 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 07:53:59,467 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 07:53:59,634 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 07:53:59,634 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 07:53:59,634 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 07:53:59,752 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 07:53:59,758 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 07:54:00,585 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 07:54:00,587 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 07:54:01,406 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 07:54:01,406 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 07:54:02,258 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 07:54:02,260 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 07:54:03,063 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 07:55:04,046 INFO [ResourceContainer.invoker.availCheck.daemon-75] (org.mc4j.ems.connection.ConnectionFactory)- Discovered libraries in 0 ms
2014-11-05 07:55:04,047 INFO [ResourceContainer.invoker.availCheck.daemon-75] (org.rhq.plugins.server.JBossAS7JMXComponent)- Loading AS7 connection [service:jmx:remoting-jmx://127.0.0.1:6999] with install path [/home/hudson/jon-server-3.3.0.CR01/jbossas/bin/client]...
2014-11-05 07:55:36,637 WARN [ConfigurationManager.threadpool-1] (rhq.core.pc.configuration.ConfigurationCheckExecutor)- Plugin Error: Invalid User resource configuration returned by Postgres plugin - Required property 'password' has a null value.
2014-11-05 07:55:36,639 WARN [ConfigurationManager.threadpool-1] (rhq.core.pc.configuration.ConfigurationCheckExecutor)- Plugin Error: Invalid User resource configuration returned by Postgres plugin - Required property 'password' has a null value.
2014-11-05 07:59:06,223 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 08:09:03,064 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 08:09:03,320 INFO [ResourceDiscoveryComponent.invoker.daemon-12] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 08:09:03,385 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 08:09:03,385 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 08:09:03,503 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 08:09:03,503 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 08:09:03,503 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 08:09:03,574 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 08:09:03,581 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 08:09:04,406 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 08:09:04,407 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 08:09:05,226 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 08:09:05,226 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 08:09:06,096 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 08:09:06,098 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 08:09:06,901 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 08:19:06,260 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 08:24:06,901 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 08:24:07,154 INFO [ResourceDiscoveryComponent.invoker.daemon-14] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 08:24:07,227 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 08:24:07,227 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 08:24:07,331 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 08:24:07,331 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 08:24:07,331 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 08:24:07,399 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 08:24:07,405 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 08:24:08,229 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 08:24:08,230 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 08:24:09,045 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 08:24:09,046 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 08:24:09,869 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 08:24:09,871 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 08:24:10,674 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 08:39:06,172 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 08:39:10,675 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 08:39:10,956 INFO [ResourceDiscoveryComponent.invoker.daemon-16] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 08:39:11,036 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 08:39:11,037 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 08:39:11,165 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 08:39:11,165 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 08:39:11,165 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 08:39:11,228 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 08:39:11,234 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 08:39:12,052 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 08:39:12,053 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 08:39:12,867 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 08:39:12,867 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 08:39:13,693 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 08:39:13,696 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 08:39:14,499 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 08:47:36,176 INFO [ResourceContainer.invoker.daemon-23] (org.rhq.plugins.jmx.MBeanResourceComponent)- Configuration management is currently disabled for the ResourceType[id=0, name=java.util.logging, plugin=RHQAgent, category=Service] Resource with key [java.util.logging:type=Logging] - it can be enabled in the Resource's Inventory > Connection tab.
2014-11-05 08:47:36,211 INFO [ResourceContainer.invoker.daemon-23] (org.rhq.plugins.jmx.MBeanResourceComponent)- Configuration management is currently disabled for the ResourceType[id=0, name=java.util.logging, plugin=RHQStorage, category=Service] Resource with key [java.util.logging:type=Logging] - it can be enabled in the Resource's Inventory > Connection tab.
2014-11-05 08:54:14,499 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 08:54:14,743 INFO [ResourceDiscoveryComponent.invoker.daemon-18] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 08:54:14,805 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 08:54:14,805 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 08:54:14,943 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 08:54:14,944 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 08:54:14,944 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 08:54:15,004 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 08:54:15,010 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 08:54:15,828 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 08:54:15,829 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 08:54:16,643 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 08:54:16,644 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 08:54:17,467 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 08:54:17,469 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 08:54:18,272 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 08:59:06,200 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 08:59:09,047 INFO [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- [PERF] Collection of measurements for [Resource[id=13703, uuid=bf47f049-0451-4ac9-a74f-5371a5f7d351, type={JBossAS7}Hibernate Persistence Unit, key=deployment=rhq.ear,subdeployment=rhq-core-domain-ejb3.jar,subsystem=jpa,hibernate-persistence-unit=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, name=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, parent=jpa]] (component=[org.rhq.modules.plugins.jbossas7.HibernatePersistenceUnitComponent@50b0ab76]) took [2038]ms
2014-11-05 08:59:39,451 WARN [ConfigurationManager.threadpool-1] (rhq.core.pc.configuration.ConfigurationCheckExecutor)- An error occurred while checking for an updated Resource configuration for Resource[id=13012, uuid=09c208af-0fd6-4567-a490-8986bfebccb0, type={Postgres}Postgres Server, key=jdbc:postgresql://127.0.0.1:5432/postgres, name=rhq, parent=fbr-ha.bc.jonqe.lab.eng.bos.redhat.com, version=8.4.20].
java.io.IOException: PostgreSQL configuration file [/var/lib/pgsql/data/postgresql.conf] does not exist or is not readable. Make sure the user the RHQ Agent is running as has read permissions on the file and its parent directory.
at org.rhq.plugins.postgres.util.PostgresqlConfFile.(PostgresqlConfFile.java:64)
at org.rhq.plugins.postgres.PostgresServerComponent.getConfigurationFile(PostgresServerComponent.java:219)
at org.rhq.plugins.postgres.PostgresServerComponent.loadResourceConfiguration(PostgresServerComponent.java:238)
at sun.reflect.GeneratedMethodAccessor78.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.rhq.core.pc.inventory.ResourceContainer$ComponentInvocation.call(ResourceContainer.java:759)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 08:59:42,868 WARN [ConfigurationManager.threadpool-1] (rhq.core.pc.configuration.ConfigurationCheckExecutor)- Plugin Error: Invalid User resource configuration returned by Postgres plugin - Required property 'password' has a null value.
2014-11-05 08:59:42,870 WARN [ConfigurationManager.threadpool-1] (rhq.core.pc.configuration.ConfigurationCheckExecutor)- Plugin Error: Invalid User resource configuration returned by Postgres plugin - Required property 'password' has a null value.
2014-11-05 09:09:18,273 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 09:09:18,492 INFO [ResourceDiscoveryComponent.invoker.daemon-20] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 09:09:18,572 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 09:09:18,572 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 09:09:18,704 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 09:09:18,704 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 09:09:18,704 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 09:09:18,754 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 09:09:18,760 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 09:09:19,579 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 09:09:19,581 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 09:09:20,395 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 09:09:20,396 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 09:09:21,222 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 09:09:21,224 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 09:09:22,026 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 09:19:06,153 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 09:19:09,024 INFO [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- [PERF] Collection of measurements for [Resource[id=13703, uuid=bf47f049-0451-4ac9-a74f-5371a5f7d351, type={JBossAS7}Hibernate Persistence Unit, key=deployment=rhq.ear,subdeployment=rhq-core-domain-ejb3.jar,subsystem=jpa,hibernate-persistence-unit=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, name=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, parent=jpa]] (component=[org.rhq.modules.plugins.jbossas7.HibernatePersistenceUnitComponent@50b0ab76]) took [2055]ms
2014-11-05 09:24:22,027 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 09:24:22,278 INFO [ResourceDiscoveryComponent.invoker.daemon-22] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 09:24:22,343 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 09:24:22,344 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 09:24:22,481 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 09:24:22,481 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 09:24:22,481 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 09:24:22,529 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 09:24:22,535 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 09:24:23,352 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 09:24:23,353 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 09:24:24,167 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 09:24:24,167 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 09:24:24,989 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 09:24:24,991 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 09:24:25,793 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 09:39:06,163 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 09:39:25,794 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 09:39:26,023 INFO [ResourceDiscoveryComponent.invoker.daemon-24] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 09:39:26,091 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 09:39:26,091 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 09:39:26,213 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 09:39:26,213 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 09:39:26,214 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 09:39:26,258 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 09:39:26,264 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 09:39:27,083 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 09:39:27,084 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 09:39:27,898 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 09:39:27,899 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 09:39:28,723 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 09:39:28,725 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 09:39:29,528 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 09:49:36,176 INFO [ResourceContainer.invoker.daemon-34] (org.rhq.plugins.jmx.MBeanResourceComponent)- Configuration management is currently disabled for the ResourceType[id=0, name=java.util.logging, plugin=RHQAgent, category=Service] Resource with key [java.util.logging:type=Logging] - it can be enabled in the Resource's Inventory > Connection tab.
2014-11-05 09:49:36,210 INFO [ResourceContainer.invoker.daemon-32] (org.rhq.plugins.jmx.MBeanResourceComponent)- Configuration management is currently disabled for the ResourceType[id=0, name=java.util.logging, plugin=RHQStorage, category=Service] Resource with key [java.util.logging:type=Logging] - it can be enabled in the Resource's Inventory > Connection tab.
2014-11-05 09:54:29,529 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 09:54:29,750 INFO [ResourceDiscoveryComponent.invoker.daemon-26] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 09:54:29,820 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 09:54:29,821 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 09:54:29,939 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 09:54:29,940 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 09:54:29,940 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 09:54:29,998 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 09:54:30,004 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 09:54:30,826 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 09:54:30,827 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 09:54:31,641 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 09:54:31,642 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 09:54:32,463 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 09:54:32,464 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 09:54:33,268 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].
2014-11-05 09:59:06,147 WARN [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Failure to collect measurement data for Resource[id=10478, uuid=d5701dc1-45d0-4dc0-b2a2-2406240379ac, type={RHQStorage}ConfigurableInternalServerMetrics, key=org.apache.cassandra.internal:type=AntiEntropySessions, name=Anti Entropy Sessions, parent=RHQ Storage Node(fbr-ha.bc.jonqe.lab.eng.bos.redhat.com)] - cause: java.lang.IllegalStateException:EMS bean was null for Resource with type [ResourceType[id=0, name=ConfigurableInternalServerMetrics, plugin=RHQStorage, category=Service]] and key [org.apache.cassandra.internal:type=AntiEntropySessions].
2014-11-05 09:59:09,077 INFO [MeasurementManager.collector-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- [PERF] Collection of measurements for [Resource[id=13703, uuid=bf47f049-0451-4ac9-a74f-5371a5f7d351, type={JBossAS7}Hibernate Persistence Unit, key=deployment=rhq.ear,subdeployment=rhq-core-domain-ejb3.jar,subsystem=jpa,hibernate-persistence-unit=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, name=rhq.ear/rhq-core-domain-ejb3.jar#rhqpu, parent=jpa]] (component=[org.rhq.modules.plugins.jbossas7.HibernatePersistenceUnitComponent@50b0ab76]) took [2118]ms
2014-11-05 10:05:39,383 WARN [ConfigurationManager.threadpool-1] (rhq.core.pc.configuration.ConfigurationCheckExecutor)- An error occurred while checking for an updated Resource configuration for Resource[id=13012, uuid=09c208af-0fd6-4567-a490-8986bfebccb0, type={Postgres}Postgres Server, key=jdbc:postgresql://127.0.0.1:5432/postgres, name=rhq, parent=fbr-ha.bc.jonqe.lab.eng.bos.redhat.com, version=8.4.20].
java.io.IOException: PostgreSQL configuration file [/var/lib/pgsql/data/postgresql.conf] does not exist or is not readable. Make sure the user the RHQ Agent is running as has read permissions on the file and its parent directory.
at org.rhq.plugins.postgres.util.PostgresqlConfFile.(PostgresqlConfFile.java:64)
at org.rhq.plugins.postgres.PostgresServerComponent.getConfigurationFile(PostgresServerComponent.java:219)
at org.rhq.plugins.postgres.PostgresServerComponent.loadResourceConfiguration(PostgresServerComponent.java:238)
at sun.reflect.GeneratedMethodAccessor78.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.rhq.core.pc.inventory.ResourceContainer$ComponentInvocation.call(ResourceContainer.java:759)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2014-11-05 10:05:42,805 WARN [ConfigurationManager.threadpool-1] (rhq.core.pc.configuration.ConfigurationCheckExecutor)- Plugin Error: Invalid User resource configuration returned by Postgres plugin - Required property 'password' has a null value.
2014-11-05 10:05:42,808 WARN [ConfigurationManager.threadpool-1] (rhq.core.pc.configuration.ConfigurationCheckExecutor)- Plugin Error: Invalid User resource configuration returned by Postgres plugin - Required property 'password' has a null value.
2014-11-05 10:09:33,269 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Executing server discovery scan...
2014-11-05 10:09:33,508 INFO [ResourceDiscoveryComponent.invoker.daemon-28] (org.rhq.plugins.postgres.PostgresDiscoveryComponent)- Discovered a postgres process: ProcessScanResult: scan=[ProcessScan: query=[process|basename|match=^(postgres|postmaster)$,process|basename|nomatch|parent=^(postgres|postmaster)$], name=[unix]], info=[process: pid=[1791], name=[/usr/bin/postmaster], ppid=[1]]
2014-11-05 10:09:33,571 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.AutoDiscoveryExecutor)- Discovered 0 new server(s).
2014-11-05 10:09:33,571 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sending [server] inventory report to Server...
2014-11-05 10:09:33,671 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform [10001]
2014-11-05 10:09:33,671 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Platform Top level services [10001]
2014-11-05 10:09:33,671 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform Top level services [10001] Local inventory changed: [false]
2014-11-05 10:09:33,712 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13011]
2014-11-05 10:09:33,715 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13011] Local inventory changed: [false]
2014-11-05 10:09:34,534 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10003]
2014-11-05 10:09:34,534 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10003] Local inventory changed: [false]
2014-11-05 10:09:35,348 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [10002]
2014-11-05 10:09:35,348 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [10002] Local inventory changed: [false]
2014-11-05 10:09:36,170 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Starting: Top Level Server [13012]
2014-11-05 10:09:36,171 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Top Level Server [13012] Local inventory changed: [false]
2014-11-05 10:09:36,976 INFO [InventoryManager.discovery-1] (rhq.core.pc.inventory.InventoryManager)- Sync Complete: Platform [10001].