wikipedia

Support Wikipedia

Monday, September 21, 2009

Bare bones JMX

Some times you wish there was a way to monitor objects in a app while it is running to check for potential problems, or just to troubleshoot a problem. Well JMX can do that and much more. JMX can also do notification for example if a Queue is full or database is out of space, things like that. But I wanted to get started with just being able to monitor an object whenever I wished to.

The example used here is an application which is basically a Socket server serving clients with data from a queue. The use case here is to look at the app and query for number of clients connected or number of objects in the queue. pretty simple huh?

Here is the code for the app.
We will use JConsole to connect to the app process and invoke the operations defined.

Now you might say that this can be done by logging the information (queue size and number of clients). True but like I said earlier using JMX gives you a way to check on things at runtime. also you could have an operation to clear the queue for a secure user (how? check for later posts!). Also why do you want to waste cpu cycles and disk space to keep logging stuff which you may or not look for.

The Java artifacts you need are an interface that declares the methods that you want to expose, a class that implements that interface. That's it. TestJMX.java is just to create and register a JMX bean.

Interface....
package com.example.testJMX;

public interface DataSocketServerMBean {
 // operations
 public int numberofClients();
 public int dataQueueSize();
}

The implementor...

package com.example.testJMX;

import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
 

 

 

 
import org.apache.log4j.Logger;

public class DataSocketServer extends Thread implements DataSocketServerMBean {
 private final static Logger log = Logger.getLogger(DataSocketServer.class);
 private final static int LISTENING_PORT = 3000;
 private ServerSocket weatherDataServer;
 public boolean shutdown = false;
 protected LinkedBlockingQueue outputDataQueue;
 private ConnectToNewClient newClient;
 private boolean byteData;
 private int listeningPort;

 public DataSocketServer() throws Exception {
  weatherDataServer = new ServerSocket(LISTENING_PORT);
  log.info("Server listening on port " + LISTENING_PORT);
  this.start();
  //start a new thread for writing data to each client socket
  newClient = new ConnectToNewClient();
 }

 public DataSocketServer(LinkedBlockingQueue dataQueue) { 
  this.outputDataQueue = dataQueue;

  try {
   weatherDataServer = new ServerSocket(LISTENING_PORT);
  } catch (IOException e) {
   log.error("Error creating ServerSocket :", e);
  }
  log.info("Server listening on port " + LISTENING_PORT);
  this.setName("WeatherDataSocketServerfor" + listeningPort);
  this.start();
  //start a new thread for writing data to each client socket
  newClient = new ConnectToNewClient();
 }

 public void run() {
  while (true) {
   try {
    log.debug("Waiting for connections.");
    if (weatherDataServer != null){
     Socket client = weatherDataServer.accept();
     log.info("Accepted a connection from: " + client.getInetAddress());
     newClient.addClient(new ClientSocket(client));
    }
    Thread.sleep(500);
   } catch (InterruptedException ix) {
    log.error("Shutting down socket server", ix);
    newClient.interrupt();
    break;
   } catch (Exception e) {
    log.error("Error accepting connections", e);
   }
  }
 }

 public void ConnectToClient(Socket clientSocket) {

 }

 /**
  * Helper class to handle each client as a separate thread.
  */
 class ConnectToNewClient extends Thread {
  // DECLARE
  private List clientSockets = Collections.synchronizedList(new ArrayList());

  // INIT
  public ConnectToNewClient() {
   this.setName("ConnectToNewClientFor" + DataSocketServer.this.listeningPort);
   this.start();
  }

  public void addClient(ClientSocket clientSocket) {
   clientSockets.add(clientSocket);
  }

  public ConnectToNewClient(ArrayList clientList) {
   this.clientSockets = clientList;
   this.setName("ConnectToNewClientFor" + DataSocketServer.this.listeningPort);
   this.start();
  }

  // ACTION
  public void run() {
   Object qobj = null;
   try {
    while (true) {
     while (clientSockets.size() > 0 && (qobj = DataSocketServer.this.outputDataQueue.peek()) != null) {
      log.info("Number of clients =" + clientSockets.size());
      //loop through each client and write to it.
      for (int i = 0; i < clientSockets.size(); i++) {
       ClientSocket clientSocket = null;
       try {
        //check if the socket is active.
        clientSocket = clientSockets.get(i);
        if (clientSocket.checkConnection()) {
         DataSocketServer.log.info("Sending data to client "
           + clientSocket.getInetAddress());
         if (DataSocketServer.this.byteData && qobj instanceof byte[])
          clientSocket.writeBytes((byte[]) qobj);
         else
          clientSocket.writeObject(qobj);
         DataSocketServer.this.outputDataQueue.remove(qobj);
        } else {
         //remove client from list
         DataSocketServer.log.info("Removing client from list : "
           + clientSocket.getInetAddress());
         clientSocket.close();
         clientSockets.remove(clientSocket);
         clientSocket = null;
        }
       } catch (RuntimeException e) {
        DataSocketServer.log.error("Error sending data to client ", e);
        //the client socket may be dead. remove it
        clientSockets.remove(clientSocket);
        clientSocket = null;
       }
      }
     }
     Thread.sleep(1000);
    }
   } catch (Exception e) {
    DataSocketServer.log.error("Error :", e);
    DataSocketServer.log.error("outputDataQueue =" + DataSocketServer.this.outputDataQueue);
    DataSocketServer.log.error("qobj = " + qobj);
   }
}
}

/**
* Socket wrapper for each client.
*/
class ClientSocket {
// DECLARE
private Socket socket = null;
private ObjectInputStream ois = null;
private OutputStream sos = null;
private ExecutorService executor;
private ObjectOutputStream oos = null;
private BufferedOutputStream bos = null;

// INIT
public ClientSocket(Socket clientSocket) {
this.socket = clientSocket;
try {
sos = socket.getOutputStream();
if (DataSocketServer.this.byteData)
bos   = new BufferedOutputStream(sos);
else
oos = new ObjectOutputStream(new BufferedOutputStream(sos));
} catch (Exception e1) {
try {
socket.close();
} catch (Exception e) {
log.error("Error creating client socket :" , e);
}
}
}

public void close() {
try {
socket.close();
if (oos != null)
oos.close();
if (bos != null)
bos.close();
} catch (Exception e) {
}
socket = null;
}

public void writeObject(Object o) {
try {
oos.flush();
if (o instanceof byte[]){
byte[] buf = (byte[])o;
bos.write(buf);
} else {
oos.writeObject(o);
}
oos.flush();
} catch (IOException e) {
// close socket.. connection maybe lost. The only way to find out is when
//we get a socketexception!!!!!!
log.error("Error writing to socket. closing socket...", e);
try {
socket.close();
} catch (IOException e1) {
}
socket = null;
} catch (Exception ex) {
log.error("Error writing data to client socket for client " + this.getInetAddress(),ex);
}
}

// UTIL
public boolean checkConnection() {
if (socket != null && !socket.isClosed() && socket.isConnected() && socket.isBound())
return true;
return false;
}

// ACCESS
public String getInetAddress() {
if (socket != null)
return socket.getInetAddress().getHostName();
return "";
}

public Socket getSocket() {
return socket;
}

public void writeBytes(byte[] buf) {
try {
if (socket != null) {
log.info("writing data " + new String(buf));
bos.flush();
bos.write(buf);
bos.flush();
}
} catch (IOException e) {
// close socket.. connection maybe lost. The only way to find out is when
//we get a socketexception!!!!!!
log.error("Error writing to socket. listening port: " + DataSocketServer.this.listeningPort + " closing socket...", e);
try {
socket.close();
} catch (IOException e1) {
}
socket = null;
} catch (Exception ex) {
log.error("Error writing data to client socket for client " + this.getInetAddress(),ex);
}
}
}

public int dataQueueSize() {
if (outputDataQueue != null)
return outputDataQueue.size();
return 0;
}

public int numberofClients() {
if (newClient != null)
return newClient.clientSockets.size();
return 0;
}

}

and finally the TestJMX class...
package com.example.testJMX;

import java.lang.management.*;
import java.util.Date;
import java.util.concurrent.LinkedBlockingQueue;

import javax.management.*;

import org.json.JSONObject;

public class Main {
    public static void main(String[] args) throws Exception {
    // Get the Platform MBean Server
    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();

    LinkedBlockingQueue testQueue = new LinkedBlockingQueue();
    JSONObject sampleObj = new JSONObject();
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    sampleObj.put("The current time on host machine", new Date());
    testQueue.add(sampleObj.toString());
    
    // Construct the ObjectName for the MBean we will register
    ObjectName name = new ObjectName("com.example.testJMX:type=DataSocketServer");

    // Create the jmx MBean
    DataSocketServer mbean = new DataSocketServer(testQueue);

    // Register the jmx MBean
    mbs.registerMBean(mbean, name);

    // Wait forever
    System.out.println("Waiting forever...");
    Thread.sleep(Long.MAX_VALUE);
    }
} 

Now run the TestJMX java class.
Launch JConsole and connect to the process
Invoking dataQueueSize and numberofClients operations...













connect to the socket server using telnet/netcat or any of those tools.
On linux ...
netcat localhost 4233
Now invoking the same operations again gives you the following results...


Tuesday, September 8, 2009

Capturing jvm Thread Dump in linux

Many a times it has happened that a Java app froze and I needed to take a thread dump to look for deadlocks or any other issues. In this case you need to perform a Thread dump on a running process. Using kill command with argument 3 produces a Thread dump, but the output goes to the stdout of the process, Which could be a console on your favorite IDE. What if the IDE is frozen too?

Well in that case using the /proc command comes in handy.
ls /proc lists all the processes running on the system currently.

my-desktop:~$ ls  /proc
1      2133   24921  3132  3615  3744         fb             partitions
10     2134   25     3135  3616  4            filesystems    sched_debug
11     2141   25214  3139  3618  4763         fs             schedstat
12     2142   26     3159  3629  5            interrupts     scsi
12087  22     26408  3163  3633  5587         iomem          self
13     2213   26426  3165  3639  6            ioports        slabinfo
14     2258   2881   3183  3645  7            irq            stat
1495   2281   29     3184  3654  722          kallsyms       swaps
15     2283   2913   3198  3656  8            kcore          sys
16     22886  2935   32    3658  856          key-users      sysrq-trigger
16653  22894  2938   3211  3664  9            kmsg           sysvipc
17     2306   2939   3240  3669  acpi         kpagecount     timer_list
18     23177  294    33    3671  asound       kpageflags     timer_stats
1865   23186  3      3316  3673  buddyinfo    latency_stats  tty
19     23191  30     3344  3674  bus          loadavg        uptime
19405  23194  3002   34    3679  cgroups      locks          version
19406  23197  30137  3466  3683  cmdline      mdstat         version_signature
19407  2330   3032   35    3685  cpuinfo      meminfo        vmallocinfo
19445  23701  3077   3536  3691  crypto       misc           vmstat
19988  2389   3080   3549  3694  devices      modules        zoneinfo
2      24     3081   36    3707  diskstats    mounts
20     2431   309    3606  3724  dma          mtrr
20798  24772  3099   3609  3728  driver       net
21     24803  31     3610  3740  execdomains  pagetypeinfo

Say we want to take a thread dump of a java process (PID 24772).  
/proc/24772/fd/1 gets you the stdout interface of the Java process where 1 is the file-descriptor for stdout.
You can capture the output to stdoutby issuing the command
cat  /proc/24772/fd/1
Now to generate a thread dump issue this from another session

kill -3  24772
You should see the thread dump output like this..

my-desktop:~$ cat  /proc/24772/fd/1
2009-09-08 10:41:19
Full thread dump Java HotSpot(TM) Client VM (14.2-b01 mixed mode, sharing):

"Low Memory Detector" daemon prio=10 tid=0x09060c00 nid=0x60d0 runnable [0x00000000]
   java.lang.Thread.State: RUNNABLE

"CompilerThread0" daemon prio=10 tid=0x0905dc00 nid=0x60ce waiting on condition [0x00000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" daemon prio=10 tid=0x0905c400 nid=0x60cd waiting on condition [0x00000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" daemon prio=10 tid=0x0904dc00 nid=0x60cc in Object.wait() [0xb5326000]
   java.lang.Thread.State: WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <0x8bcd8918> (a java.lang.ref.ReferenceQueue$Lock)
    at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
    - locked <0x8bcd8918> (a java.lang.ref.ReferenceQueue$Lock)
    at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
    at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)

"Reference Handler" daemon prio=10 tid=0x09049400 nid=0x60ca in Object.wait() [0xb5377000]
   java.lang.Thread.State: WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <0x8bcd89a0> (a java.lang.ref.Reference$Lock)
    at java.lang.Object.wait(Object.java:485)
    at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
    - locked <0x8bcd89a0> (a java.lang.ref.Reference$Lock)

"main" prio=10 tid=0x09023400 nid=0x60c6 runnable [0xb75e9000]
   java.lang.Thread.State: RUNNABLE
    at java.net.PlainSocketImpl.socketAccept(Native Method)
    at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
    - locked <0x8b8364b8> (a java.net.SocksSocketImpl)
    at java.net.ServerSocket.implAccept(ServerSocket.java:453)
    at java.net.ServerSocket.accept(ServerSocket.java:421)
    at org.coastal.log4jext.SocketServerAsWinService.main(SocketServerAsWinService.java:41)

"VM Thread" prio=10 tid=0x09047800 nid=0x60c9 runnable 

"VM Periodic Task Thread" prio=10 tid=0x09062800 nid=0x60d1 waiting on condition 

JNI global references: 945

Heap
 def new generation   total 960K, used 488K [0x8b7d0000, 0x8b8d0000, 0x8bcb0000)
  eden space 896K,  47% used [0x8b7d0000, 0x8b83a3b8, 0x8b8b0000)
  from space 64K, 100% used [0x8b8c0000, 0x8b8d0000, 0x8b8d0000)
  to   space 64K,   0% used [0x8b8b0000, 0x8b8b0000, 0x8b8c0000)
 tenured generation   total 4096K, used 467K [0x8bcb0000, 0x8c0b0000, 0x8f7d0000)
   the space 4096K,  11% used [0x8bcb0000, 0x8bd24f00, 0x8bd25000, 0x8c0b0000)
 compacting perm gen  total 12288K, used 373K [0x8f7d0000, 0x903d0000, 0x937d0000)
   the space 12288K,   3% used [0x8f7d0000, 0x8f82d6a0, 0x8f82d800, 0x903d0000)
    ro space 8192K,  74% used [0x937d0000, 0x93dca2a8, 0x93dca400, 0x93fd0000)
    rw space 12288K,  59% used [0x93fd0000, 0x946e7878, 0x946e7a00, 0x94bd0000)