Bottom line:

Some server generated events do not get delivered to the client in production over websockets. However, the websocket connections get established just fine.

Case study:

I open Google Chrome and connect to our server. Open the devtools. Under WS tab, I see that connection got established fine, but I receive no frames when, let’s say, server needs to update something on the page. I wait for a while and sometimes (only sometimes) I get some events with a huge amount of delay. This works as expected locally however.

Question:

Has anyone seen similar websocket behavior and has any suggestions on how to eliminate variables for this investigation.

Infrastructure:

Server: Linux Tomcat

Two servers that handle: 1. Traffic from Devices (Communicating over TCP/IP with the Server) 2. Traffic from Users

Users and Devices are many to many relationship. If a user gets connected to a server that doesn’t have a device connected. This server looks on the other server and handles the info exchange.

There is a firewall in front of the servers.

Code:

https://github.com/kino6052/websockets-issue

WebSocketServerEndpoint.java

@ServerEndpoint("/actions") public class WebSocketServerEndpoint { static private final org.slf4j.Logger logger = LoggerFactory.getLogger(WebSocketServerEndpoint.class); @OnOpen public void open(Session session) { WebSocketSessionHandler.addSession(session); } @OnClose public void close(Session session) { WebSocketSessionHandler.removeSession(session); } @OnError public void onError(Throwable error) { //Logger.getLogger(WebSocketServerEndpoint.class.getName()).log(Level.SEVERE, null, error); } @OnMessage public void handleMessage(String message, Session session) { try (JsonReader reader = Json.createReader(new StringReader(message))) { JsonObject jsonMessage = reader.readObject(); Long userId = null; Long tenantId = null; switch (WebSocketActions.valueOf(jsonMessage.getString("action"))){ case SaveUserId: userId = getUserId(jsonMessage); tenantId = getTenantId(jsonMessage); Long userIdKey = WebSocketSessionHandler.saveUserId(userId, session); Long tenantUserKey = WebSocketSessionHandler.saveTenantUser(tenantId, userId); WebSocketSessionHandler.updateUserSessionKeys(session, tenantUserKey, userIdKey); // Needed for Making Weak Maps Keep Their Keys if Session is Currently Active } } catch (Exception e) { logger.error(e.toString()); } } private Long getUserId(JsonObject jsonMessage) { Long userId = null; try { userId = Long.parseLong(((Integer) jsonMessage.getInt("userId")).toString()); return userId; } catch (Exception e) { logger.error(e.getMessage()); return userId; } } private Long getTenantId(JsonObject jsonMessage) { Long tenantId = null; try { tenantId = Long.parseLong(((Integer) jsonMessage.getInt("tenantId")).toString()); return tenantId; } catch (Exception e) { logger.error(e.getMessage()); return tenantId; } } }

WebSocketService.java

@Singleton public class WebSocketService { private static final Logger logger = LoggerFactory.getLogger(WebSocketService.class); public enum WebSocketEvents{ OnConnection, OnActivity, OnAccesspointStatus, OnClosedStatus, OnConnectedStatus, OnAlert, OnSessionExpired } public enum WebSocketActions{ SaveUserId } @WebPost("/lookupWebSocketSessions") public WebResponse lookupWebSocketSessions(@JsonArrayParam("userIds") List<Integer> userIds, @WebParam("message") String message){ try { for (Integer userIdInt : userIds) { Long userId = Long.parseLong(userIdInt.toString()); if (WebSocketSessionHandler.sendToUser(userId, message) == 0) { } else { //logger.debug("Couldn't Send to User"); } } } catch (ClassCastException e) { //logger.error(e.getMessage()); return webResponseBuilder.fail(e); } catch (Exception e) { //logger.error(e.getMessage()); return webResponseBuilder.fail(e); } return webResponseBuilder.success(message); } @WebPost("/lookupWebSocketHistorySessions") public WebResponse lookupWebSocketHistorySessions(@JsonArrayParam("userIds") List<Integer> userIds, @WebParam("message") String message){ try { for (Integer userIdInt : userIds) { Long userId = Long.parseLong(userIdInt.toString()); if (WebSocketHistorySessionHandler.sendToUser(userId, message) == 0) { } else { //logger.debug("Couldn't Send to User"); } } } catch (ClassCastException e) { //logger.error(e.getMessage()); return webResponseBuilder.fail(e); } catch (Exception e) { //logger.error(e.getMessage()); return webResponseBuilder.fail(e); } return webResponseBuilder.success(message); } // Kick Out a User if Their Session is no Longer Valid public void sendLogout(User user) { try { Long userId = user.getId(); List<Long> userIds = new ArrayList<>(); userIds.add(userId); JSONObject result = new JSONObject(); result.put("userId", userId); JSON message = WebSocketSessionHandler.createMessage(WebSocketEvents.OnSessionExpired, result); lookOnOtherServers(userIds, message); } catch (Exception e) { logger.error("Couldn't Logout User"); } } // Send History after Processing Data // Returns "0" if success, "-1" otherwise public int sendHistory(Activity activity) { try { TimezoneService.TimeZoneConfig timeZoneConfig = timezoneService.getTimezoneConfigsByAp(null, activity.getAccesspointId()); JSONObject result = (JSONObject) JSONSerializer.toJSON(activity); String timezoneId = timezoneService.convertTimezoneConfigToTimezoneId(timeZoneConfig); result.put("timezoneString", timezoneId); result.put( "profileId", userDao.getUserProfileId(activity.getUserId()) ); JSON message = WebSocketHistorySessionHandler.createMessage(WebSocketEvents.OnActivity, result); List<Long> userIds = getUsersSubscribedToActivity(activity.getTenantId()); lookOnOtherServersHistory(userIds, message); return 0; } catch (Exception e) { //logger.error("Couldn't Send History"); return -1; } } // SendAlertUpdate after Processing Data public void sendAlertUpdate(Alert alert) { try { List<Long> userIds = getUsersUnderTenantByAccesspointId(alert.getAccesspointId()); JSONObject result = JSONObject.fromObject(alert); JSON message = WebSocketSessionHandler.createMessage(WebSocketEvents.OnAlert, result); lookOnOtherServers(userIds, message); } catch (Exception e) { //logger.error("Couldn't Send Aleart"); } } // Send Connected Status after Processing Data public void sendConnectedStatus(Long accesspointId, Boolean isConnected) { try { List<Long> userIds = getUsersUnderTenantByAccesspointId(accesspointId); JSONObject result = new JSONObject(); result.put("accesspointId", accesspointId); result.put("isConnected", isConnected); JSON message = WebSocketSessionHandler.createMessage(WebSocketEvents.OnConnectedStatus, result); lookOnOtherServers(userIds, message); } catch (Exception e) { //logger.error("Couldn't Send Connected Status"); } } public int sendHistory(CredentialActivity activity) { try { TimezoneService.TimeZoneConfig timeZoneConfig = timezoneService.getTimezoneConfigsByAp(null, activity.getAccesspointId()); JSONObject result = (JSONObject) JSONSerializer.toJSON(activity); String timezoneId = timezoneService.convertTimezoneConfigToTimezoneId(timeZoneConfig); result.put("timezoneString", timezoneId); result.put( "profileId", userDao.getUserProfileId(activity.getUserId()) ); JSON message = WebSocketHistorySessionHandler.createMessage(WebSocketEvents.OnActivity, result); List<Long> userIds = getUsersUnderTenantByAccesspointId(activity.getAccesspointId()); lookOnOtherServersHistory(userIds, message); return 0; } catch (Exception e) { return -1; } } public Boolean isUserSessionAvailable(Long id) { return WebSocketSessionHandler.isUserSessionAvailable(id); } public void lookOnOtherServers(List<Long> userId, JSON data){ List<String> urls = awsService.getServerURLs(); for (String url : urls) { postJSONDataToUrl(url, userId, data); } } public void lookOnOtherServersHistory(List<Long> userId, JSON data){ List<String> urls = awsService.getServerURLsHistory(); for (String url : urls) { postJSONDataToUrl(url, userId, data); } } public int sendClosedStatus(AccesspointStatus accesspointStatus){ try { JSONObject accesspointStatusJSON = new JSONObject(); accesspointStatusJSON.put("accesspointId", accesspointStatus.getAccesspointId()); accesspointStatusJSON.put("openStatus", accesspointStatus.getOpenStatus()); List<Long> userIds = getUsersUnderTenantByAccesspointId(accesspointStatus.getAccesspointId()); lookOnOtherServers(userIds, accesspointStatusJSON); return 0; } catch (Exception e) { return -1; } } public List<Long> getUsersSubscribedToActivity(Long tenantId) { List<Long> userList = WebSocketSessionHandler.getUsersForTenant(tenantId); return userList; } private List<Long> getUsersUnderTenantByAccesspointId(Long accesspointId) { List<Long> userList = new ArrayList<>(); User user = userDao.getBackgroundUserByAccesspoint(accesspointId); List<Record> recordList = tenantDao.getTenantsByUser(user, user.getId()); for (Record record : recordList) { Long tenantId = (Long) record.get("id"); userList.addAll(getUsersSubscribedToActivity(tenantId)); } return userList; } public void postJSONDataToUrl(String url, List<Long> userId, JSON data) throws AppException { List<NameValuePair> parameters; HttpResponse httpResponse; HttpClientService.SimpleHttpClient simpleHttpClient = httpClientService.createHttpClient(url); try { parameters = httpClientService.convertJSONObjectToNameValuePair(userId, data); } catch (Exception e) { throw new AppException("Couldn't Convert Input Parameters"); } try { httpResponse = simpleHttpClient.sendHTTPPost(parameters); } catch (Exception e) { throw new AppException("Couldn't Get Data from the Server"); } if (httpResponse == null) { throw new AppException("Couldn't Send to Another Server"); } else { //logger.error(httpResponse.getStatusLine().toString()); } } }

WebSocketSessionHandler.java