So I am working on a MEAN stack project where in node I will emit the data to UI and in frontend I had some widgets ( as charts like gauge , line , pie and table ) where user can add dynamically whatever widget they need , now in backend I use reddis , socket io and nodejs , currently my code works fine but I want to optimize it as what if multiple devices connected and alot of payloads received , this function will hit through postman api for now ,
What I tried was i store the dashboard IDs that are connected in socket room and in save action I sums the data values of existing fields and make it as one , and used reddis to cache some data and get that if not then fetch from db and removed some .map() and used normal for loops
And I use worker file for inserting the reddis saved data in monogodb
My main data emit data nodejs file :
const { dbOperation } = require("../helper/db_operation");
const catchAsync = require("../util/cath_async");
const { getDeviceCollection } = require("../util/db");
const { ObjectId } = require("mongodb");
const { insertDataToRedis, InsertDataForTriggersToRedis } = require("../redis/RedisData.js");
const { client } = require("../redis/redisConnect.js");
exports.iotDeviceSaveAction = catchAsync(async (req, userdata, db, next, result) => {
const data = req.body.data;
const fk_device_id = req.body.fk_device_id;
if (!fk_device_id) {
return result("Device ID Is Required", 400);
}
if (!data) {
return result("Invalid Data", 400);
}
try {
if (Array.isArray(data) && data?.length == 0) return result("Data is empty", 200);
const deviceKey = `device:${fk_device_id}`;
const isCollectionExists_in_redis = await client.get(deviceKey);
let deviceCollection = isCollectionExists_in_redis;
if (!deviceCollection) {
const deviceModel = await db.collection("manage-device").findOne(
{ _id: new ObjectId(fk_device_id) },
{
projection: { fk_collection_name: 1 },
}
);
if (!deviceModel) {
return result("Device not found", 400);
}
if (!deviceModel.fk_collection_name) {
return result("Device collection not found", 400);
}
deviceCollection = deviceModel.fk_collection_name;
await client.set(deviceKey, deviceCollection);
}
const now = new Date();
const hashmap = data.reduce((acc, item) => {
for (const [key, value] of Object.entries(item)) {
if (typeof value === "number" && !isNaN(value)) {
acc[key] = (acc[key] || 0) + value;
} else {
acc[key] = value;
}
}
return acc;
}, {});
hashmap.fk_device_id = fk_device_id;
hashmap.timestamp = now;
const formattedData = [hashmap];
const data_for_batch_save = {
insert_data: formattedData,
db_name: userdata.db_name,
};
const data_for_triggers = {
insert_data: [hashmap],
db_name: userdata.db_name,
};
InsertDataForTriggersToRedis(data_for_triggers);
insertDataToRedis(data_for_batch_save);
const socketIDS = Array.from(global.socket.sockets.sockets.values());
const socketsWithDB = socketIDS.filter((socket) => socket.dbID);
if (socketsWithDB.length <= 0) return result("Dashboard not found", 400);
const widgetsWithKeys = [];
for (let i = 0; i < socketsWithDB.length; i++) {
const DashboardID = socketsWithDB[i].dbID;
const dashboardKey = `dashboard:${DashboardID}`;
let dashboardCollection = await client.get(dashboardKey);
if (!dashboardCollection) {
const dashboardModel = await db.collection("manage-dashboard").findOne({ _id: new ObjectId(DashboardID) }, { projection: { widgets: 1 } });
if (!dashboardModel) {
continue;
}
if (!Array.isArray(dashboardModel.widgets)) {
dashboardModel.widgets = [];
}
await client.set(dashboardKey, JSON.stringify(dashboardModel.widgets));
} else {
dashboardCollection = JSON.parse(dashboardCollection);
}
const getWidgetsBasedOnDeviceID = Array.isArray(dashboardCollection) ? dashboardCollection.filter((widget) => widget.deviceID === deviceCollection) : [];
if (getWidgetsBasedOnDeviceID.length <= 0) return result("Widgets not found", 400);
for (let k = 0; k < getWidgetsBasedOnDeviceID.length; k++) {
const widget = getWidgetsBasedOnDeviceID[k];
const widgetId = widget._id.toString();
const values = [];
if (widget.type === "bar-chart") {
if (!widget.widgetOptions || !widget.widgetOptions.labelValuePairs || widget.widgetOptions.labelValuePairs.length <= 0 || widget.widgetOptions.labelValuePairs[0].label === widget.widgetOptions.xAxis) {
continue;
}
}
if (widget.type === "stack-chart") {
if (!widget.widgetOptions || !widget.widgetOptions.labelValuePairs || widget.widgetOptions.labelValuePairs.length <= 0) {
continue;
}
}
for (let i = 0; i < formattedData.length; i++) {
const dataItem = formattedData[i];
if (!dataItem) continue;
if (widget.type === "table") {
values.push(dataItem);
} else if (widget.type === "gauge-chart") {
const valueField = widget.widgetOptions?.valueField;
if (valueField && dataItem[valueField] !== undefined) {
values.push({
value: dataItem[valueField],
});
}
} else if (widget.type === "pie-chart" || widget.type === "line-chart") {
if (widget.widgetOptions?.labelValuePairs) {
for (let a = 0; a < widget.widgetOptions.labelValuePairs.length; a++) {
const pair = widget.widgetOptions.labelValuePairs[a];
if (dataItem[pair.value] !== undefined) {
values.push({
key: dataItem[pair.label] ?? pair.label,
value: dataItem[pair.value],
});
}
}
}
} else if (widget.type === "bar-chart") {
for (let b = 0; b < widget.widgetOptions.labelValuePairs.length; b++) {
const pair = widget.widgetOptions.labelValuePairs[b];
if (dataItem[pair.value] !== undefined) {
values.push({
value: dataItem[pair.value],
key: String(pair.value),
xAxis: dataItem[widget.widgetOptions.xAxis],
});
}
}
} else if (widget.type === "stack-chart") {
for (let c = 0; c < widget.widgetOptions.labelValuePairs.length; c++) {
const pair = widget.widgetOptions.labelValuePairs[c];
if (dataItem[pair.value] !== undefined) {
values.push({
value: dataItem[pair.value],
key: String(pair.label),
xAxis: dataItem[widget.widgetOptions.xAxis],
});
}
}
}
}
widgetsWithKeys.push({
widgetId: widgetId,
values: values,
});
}
}
if (widgetsWithKeys.length > 0 && socketsWithDB.length > 0) {
global.socket.emit("newData", {
Widgetdata: widgetsWithKeys,
});
}
return result("Success", 200);
} catch (error) {
console.log("error", error);
return result(error.message || "Error", 400);
}
});
My socket handler :
const socketIO = require("socket.io");
const { createAdapter } = require("@socket.io/redis-adapter");
const { connectRedis } = require("../redis/redisConnect.js");
async function setUpSocket(server) {
const socket = socketIO(server, {
cors: {
origin: "\*",
},
});
const pubClient = await connectRedis();
const subClient = await pubClient.duplicate().connect();
socket.adapter(createAdapter(pubClient, subClient));
global.socket = socket;
socket.on("connection", (socket) =\> {
console.log("Socket connected", socket.id);
socket.on("disconnect", () =\> {
console.log("Socket disconnected", socket.id);
});
socket.on("dashboard-socket", (dbID) =\> {
if (!dbID) return;
if (socket.dbID) {
socket.leave(socket.dbID);
console.log(\`Socket ${socket.id} left room: ${socket.dbID}\`);
}
socket.join(dbID);
socket.dbID = dbID;
console.log(\`Socket ${socket.id} joined room: ${dbID}\`);
});
socket.on("error", (err) =\> {
console.error("Unhandled socket error:", err);
});
});
}
module.exports = setUpSocket;
Any advice or help would be appreciated , Thank you
