APPLIES TO:
Oracle WebLogic Server - Version 10.0 to 10.3.6
Information in this document applies to any platform.
SYMPTOMS
The WLST connect command hangs up because of a Java VM ClassLoader deadlock issue. You will see the following thread stacks in the thread dumps. The "ExecuteThread: '3' for queue: 'default'" thread waiting to lock <6a04a378>. The "main" thread has lock <6a04a378>. And "main" thread waiting to lock <69a506c0>. The "ExecuteThread: '3' for queue: 'default'" thread has lock <69a506c0>.
"ExecuteThread: '3' for queue: 'default'" daemon prio=7 tid=0049b500 nid=17 lwp_id=4192481 waiting for monitor entry [358ff000..35900d50]
at java.lang.ClassLoader.getPackage(ClassLoader.java:1441)
- waiting to lock <6a04a378> (a java.util.HashMap)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:220)
at java.net.URLClassLoader.access$100(URLClassLoader.java:56)
at java.net.URLClassLoader$1.run(URLClassLoader.java:195)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:188)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
- locked <69a506c0> (a sun.misc.Launcher$AppClassLoader)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:268)
- locked <69a506c0> (a sun.misc.Launcher$AppClassLoader)
at java.lang.ClassLoader.loadClass(ClassLoader.java:251)
at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:319)
- locked <69a506c0> (a sun.misc.Launcher$AppClassLoader)
at weblogic.timers.internal.TimerImpl.recordExpirationTime(TimerImpl.java:347)
at weblogic.timers.internal.TimerImpl.run(TimerImpl.java:272)
at weblogic.work.ExecuteRequestAdapter.execute(ExecuteRequestAdapter.java:21)
at weblogic.kernel.ExecuteThread.execute(ExecuteThread.java:145)
at weblogic.kernel.ExecuteThread.run(ExecuteThread.java:117)
...
"main" prio=7 tid=004999e0 nid=1 lwp_id=-1 waiting for monitor entry [7fffa000..7fffd6e8]
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:261)
- waiting to lock <69a506c0> (a sun.misc.Launcher$AppClassLoader)
at java.lang.ClassLoader.loadClass(ClassLoader.java:251)
at java.net.URL.getURLStreamHandler(URL.java:1141)
at java.net.URL.<init>(URL.java:393)
at java.net.URL.<init>(URL.java:283)
at java.net.URL.<init>(URL.java:306)
at sun.net.www.ParseUtil.fileToEncodedURL(ParseUtil.java:261)
at java.lang.Package$1.run(Package.java:514)
at java.security.AccessController.doPrivileged(Native Method)
at java.lang.Package.defineSystemPackage(Package.java:505)
at java.lang.Package.getSystemPackage(Package.java:481)
- locked <69a63cb8> (a java.util.HashMap)
at java.lang.ClassLoader.getPackage(ClassLoader.java:1447)
- locked <6a05d300> (a java.util.HashMap)
at java.lang.ClassLoader.getPackage(ClassLoader.java:1445)
- locked <6a04a378> (a java.util.HashMap)
at java.lang.ClassLoader.getPackage(ClassLoader.java:1445)
- locked <69a54780> (a java.util.HashMap)
at java.lang.ClassLoader.getPackage(ClassLoader.java:1445)
- locked <69d566e0> (a java.util.HashMap)
at weblogic.utils.classloaders.GenericClassLoader.definePackage(GenericClassLoader.java:442)
at weblogic.utils.classloaders.GenericClassLoader.defineClass(GenericClassLoader.java:335)
at weblogic.utils.classloaders.GenericClassLoader.findLocalClass(GenericClassLoader.java:291)
- locked <69cddb50> (a weblogic.utils.classloaders.GenericClassLoader)
at weblogic.utils.classloaders.GenericClassLoader.findClass(GenericClassLoader.java:259)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
- locked <69cddb50> (a weblogic.utils.classloaders.GenericClassLoader)
at java.lang.ClassLoader.loadClass(ClassLoader.java:251)
at weblogic.utils.classloaders.GenericClassLoader.loadClass(GenericClassLoader.java:179)
at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:319)
- locked <69cddb50> (a weblogic.utils.classloaders.GenericClassLoader)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:246)
at weblogic.utils.classloaders.GenericClassLoader.defineCodeGenClass(GenericClassLoader.java:503)
at weblogic.utils.classfile.utils.CodeGenerator.generateClass(CodeGenerator.java:73)
at weblogic.rmi.internal.StubGenerator.hotCodeGenClass(StubGenerator.java:775)
at weblogic.rmi.internal.StubGenerator.getStubClass(StubGenerator.java:759)
at weblogic.rmi.internal.StubGenerator.generateStub(StubGenerator.java:786)
at weblogic.rmi.internal.StubGenerator.generateStub(StubGenerator.java:779)
at weblogic.rmi.extensions.StubFactory.getStub(StubFactory.java:76)
at weblogic.rmi.internal.StubInfo.resolveObject(StubInfo.java:214)
at weblogic.rmi.internal.StubInfo.readResolve(StubInfo.java:208)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at java.io.ObjectStreamClass.invokeReadResolve(ObjectStreamClass.java:1033)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1728)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1305)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:348)
at weblogic.utils.io.ChunkedObjectInputStream.readObject(ChunkedObjectInputStream.java:195)
at weblogic.rjvm.MsgAbbrevInputStream.readObject(MsgAbbrevInputStream.java:565)
at weblogic.utils.io.ChunkedObjectInputStream.readObject(ChunkedObjectInputStream.java:191)
at weblogic.rmi.internal.ObjectIO.readObject(ObjectIO.java:62)
at weblogic.rjvm.ResponseImpl.unmarshalReturn(ResponseImpl.java:227)
at weblogic.rmi.internal.BasicRemoteRef.invoke(BasicRemoteRef.java:224)
at weblogic.management.remote.iiop.IIOPServerImpl_1001_WLStub.newClient(Unknown Source)
at weblogic.management.remote.common.RMIServerWrapper.newClient(ClientProviderBase.java:234)
at javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2238)
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:271)
- locked <65b39430> (a weblogic.management.remote.common.WLSRMIConnector)
at weblogic.management.remote.common.WLSRMIConnector.doConnect(WLSRMIConnector.java:146)
- locked <65b39430> (a weblogic.management.remote.common.WLSRMIConnector)
at weblogic.management.remote.common.WLSRMIConnector.access$100(WLSRMIConnector.java:24)
at weblogic.management.remote.common.WLSRMIConnector$2.run(WLSRMIConnector.java:134)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:363)
at weblogic.security.service.SecurityManager.runAs(Unknown Source)
at weblogic.security.Security.runAs(Security.java:61)
at weblogic.management.remote.common.WLSRMIConnector.connect(WLSRMIConnector.java:132)
- locked <65b39430> (a weblogic.management.remote.common.WLSRMIConnector)
at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:248)
at weblogic.management.scripting.WLSTHelper.lookupMBeanServerConnection(WLSTHelper.java:656)
at weblogic.management.scripting.WLSTHelper.initRuntimeServerConnection(WLSTHelper.java:376)
at weblogic.management.scripting.WLSTHelper.initConnections(WLSTHelper.java:305)
at weblogic.management.scripting.WLSTHelper.connect(WLSTHelper.java:201)
at weblogic.management.scripting.WLScriptContext.connect(WLScriptContext.java:60)
at weblogic.management.scripting.utils.WLSTUtil.initializeOnlineWLST(WLSTUtil.java:121)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.python.core.PyReflectedFunction.__call__(PyReflectedFunction.java:160)
at org.python.core.PyMethod.__call__(PyMethod.java:96)
at org.python.core.PyObject.__call__(PyObject.java:248)
at org.python.core.PyObject.invoke(PyObject.java:2016)
at org.python.pycode._pyx6.connect$1(<iostream>:16)
at org.python.pycode._pyx6.call_function(<iostream>)
at org.python.core.PyTableCode.call(PyTableCode.java:208)
at org.python.core.PyTableCode.call(PyTableCode.java:404)
at org.python.core.PyFunction.__call__(PyFunction.java:184)
at org.python.pycode._pyx18.f$0(/opt/bea/user_projects/domains/test-domain/shutdown.py:1)
at org.python.pycode._pyx18.call_function(/opt/bea/user_projects/domains/test-domain/shutdown.py)
at org.python.core.PyTableCode.call(PyTableCode.java:208)
at org.python.core.PyCode.call(PyCode.java:14)
at org.python.core.Py.runCode(Py.java:1135)
at org.python.util.PythonInterpreter.execfile(PythonInterpreter.java:167)
at weblogic.management.scripting.WLST.main(WLST.java:106)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at weblogic.WLST.main(WLST.java:29)
at java.lang.ClassLoader.getPackage(ClassLoader.java:1441)
- waiting to lock <6a04a378> (a java.util.HashMap)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:220)
at java.net.URLClassLoader.access$100(URLClassLoader.java:56)
at java.net.URLClassLoader$1.run(URLClassLoader.java:195)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:188)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
- locked <69a506c0> (a sun.misc.Launcher$AppClassLoader)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:268)
- locked <69a506c0> (a sun.misc.Launcher$AppClassLoader)
at java.lang.ClassLoader.loadClass(ClassLoader.java:251)
at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:319)
- locked <69a506c0> (a sun.misc.Launcher$AppClassLoader)
at weblogic.timers.internal.TimerImpl.recordExpirationTime(TimerImpl.java:347)
at weblogic.timers.internal.TimerImpl.run(TimerImpl.java:272)
at weblogic.work.ExecuteRequestAdapter.execute(ExecuteRequestAdapter.java:21)
at weblogic.kernel.ExecuteThread.execute(ExecuteThread.java:145)
at weblogic.kernel.ExecuteThread.run(ExecuteThread.java:117)
...
"main" prio=7 tid=004999e0 nid=1 lwp_id=-1 waiting for monitor entry [7fffa000..7fffd6e8]
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:261)
- waiting to lock <69a506c0> (a sun.misc.Launcher$AppClassLoader)
at java.lang.ClassLoader.loadClass(ClassLoader.java:251)
at java.net.URL.getURLStreamHandler(URL.java:1141)
at java.net.URL.<init>(URL.java:393)
at java.net.URL.<init>(URL.java:283)
at java.net.URL.<init>(URL.java:306)
at sun.net.www.ParseUtil.fileToEncodedURL(ParseUtil.java:261)
at java.lang.Package$1.run(Package.java:514)
at java.security.AccessController.doPrivileged(Native Method)
at java.lang.Package.defineSystemPackage(Package.java:505)
at java.lang.Package.getSystemPackage(Package.java:481)
- locked <69a63cb8> (a java.util.HashMap)
at java.lang.ClassLoader.getPackage(ClassLoader.java:1447)
- locked <6a05d300> (a java.util.HashMap)
at java.lang.ClassLoader.getPackage(ClassLoader.java:1445)
- locked <6a04a378> (a java.util.HashMap)
at java.lang.ClassLoader.getPackage(ClassLoader.java:1445)
- locked <69a54780> (a java.util.HashMap)
at java.lang.ClassLoader.getPackage(ClassLoader.java:1445)
- locked <69d566e0> (a java.util.HashMap)
at weblogic.utils.classloaders.GenericClassLoader.definePackage(GenericClassLoader.java:442)
at weblogic.utils.classloaders.GenericClassLoader.defineClass(GenericClassLoader.java:335)
at weblogic.utils.classloaders.GenericClassLoader.findLocalClass(GenericClassLoader.java:291)
- locked <69cddb50> (a weblogic.utils.classloaders.GenericClassLoader)
at weblogic.utils.classloaders.GenericClassLoader.findClass(GenericClassLoader.java:259)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
- locked <69cddb50> (a weblogic.utils.classloaders.GenericClassLoader)
at java.lang.ClassLoader.loadClass(ClassLoader.java:251)
at weblogic.utils.classloaders.GenericClassLoader.loadClass(GenericClassLoader.java:179)
at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:319)
- locked <69cddb50> (a weblogic.utils.classloaders.GenericClassLoader)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:246)
at weblogic.utils.classloaders.GenericClassLoader.defineCodeGenClass(GenericClassLoader.java:503)
at weblogic.utils.classfile.utils.CodeGenerator.generateClass(CodeGenerator.java:73)
at weblogic.rmi.internal.StubGenerator.hotCodeGenClass(StubGenerator.java:775)
at weblogic.rmi.internal.StubGenerator.getStubClass(StubGenerator.java:759)
at weblogic.rmi.internal.StubGenerator.generateStub(StubGenerator.java:786)
at weblogic.rmi.internal.StubGenerator.generateStub(StubGenerator.java:779)
at weblogic.rmi.extensions.StubFactory.getStub(StubFactory.java:76)
at weblogic.rmi.internal.StubInfo.resolveObject(StubInfo.java:214)
at weblogic.rmi.internal.StubInfo.readResolve(StubInfo.java:208)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at java.io.ObjectStreamClass.invokeReadResolve(ObjectStreamClass.java:1033)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1728)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1305)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:348)
at weblogic.utils.io.ChunkedObjectInputStream.readObject(ChunkedObjectInputStream.java:195)
at weblogic.rjvm.MsgAbbrevInputStream.readObject(MsgAbbrevInputStream.java:565)
at weblogic.utils.io.ChunkedObjectInputStream.readObject(ChunkedObjectInputStream.java:191)
at weblogic.rmi.internal.ObjectIO.readObject(ObjectIO.java:62)
at weblogic.rjvm.ResponseImpl.unmarshalReturn(ResponseImpl.java:227)
at weblogic.rmi.internal.BasicRemoteRef.invoke(BasicRemoteRef.java:224)
at weblogic.management.remote.iiop.IIOPServerImpl_1001_WLStub.newClient(Unknown Source)
at weblogic.management.remote.common.RMIServerWrapper.newClient(ClientProviderBase.java:234)
at javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2238)
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:271)
- locked <65b39430> (a weblogic.management.remote.common.WLSRMIConnector)
at weblogic.management.remote.common.WLSRMIConnector.doConnect(WLSRMIConnector.java:146)
- locked <65b39430> (a weblogic.management.remote.common.WLSRMIConnector)
at weblogic.management.remote.common.WLSRMIConnector.access$100(WLSRMIConnector.java:24)
at weblogic.management.remote.common.WLSRMIConnector$2.run(WLSRMIConnector.java:134)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:363)
at weblogic.security.service.SecurityManager.runAs(Unknown Source)
at weblogic.security.Security.runAs(Security.java:61)
at weblogic.management.remote.common.WLSRMIConnector.connect(WLSRMIConnector.java:132)
- locked <65b39430> (a weblogic.management.remote.common.WLSRMIConnector)
at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:248)
at weblogic.management.scripting.WLSTHelper.lookupMBeanServerConnection(WLSTHelper.java:656)
at weblogic.management.scripting.WLSTHelper.initRuntimeServerConnection(WLSTHelper.java:376)
at weblogic.management.scripting.WLSTHelper.initConnections(WLSTHelper.java:305)
at weblogic.management.scripting.WLSTHelper.connect(WLSTHelper.java:201)
at weblogic.management.scripting.WLScriptContext.connect(WLScriptContext.java:60)
at weblogic.management.scripting.utils.WLSTUtil.initializeOnlineWLST(WLSTUtil.java:121)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.python.core.PyReflectedFunction.__call__(PyReflectedFunction.java:160)
at org.python.core.PyMethod.__call__(PyMethod.java:96)
at org.python.core.PyObject.__call__(PyObject.java:248)
at org.python.core.PyObject.invoke(PyObject.java:2016)
at org.python.pycode._pyx6.connect$1(<iostream>:16)
at org.python.pycode._pyx6.call_function(<iostream>)
at org.python.core.PyTableCode.call(PyTableCode.java:208)
at org.python.core.PyTableCode.call(PyTableCode.java:404)
at org.python.core.PyFunction.__call__(PyFunction.java:184)
at org.python.pycode._pyx18.f$0(/opt/bea/user_projects/domains/test-domain/shutdown.py:1)
at org.python.pycode._pyx18.call_function(/opt/bea/user_projects/domains/test-domain/shutdown.py)
at org.python.core.PyTableCode.call(PyTableCode.java:208)
at org.python.core.PyCode.call(PyCode.java:14)
at org.python.core.Py.runCode(Py.java:1135)
at org.python.util.PythonInterpreter.execfile(PythonInterpreter.java:167)
at weblogic.management.scripting.WLST.main(WLST.java:106)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at weblogic.WLST.main(WLST.java:29)
CAUSE
This is a Java VM issue which is documented at JDK-4670071 : java.lang.ClassLoader.loadClassInternal(String) is too restrictive.
The Problem was fixed by the following versions.
- Java 7 GA
- Java 6 update 18
This issue does not occur on Java 7 and higher.