mongodb_server.py
#! /bin/env python #-*- coding:utf8 -*- import sys import os from bson.timestamp import Timestamp import pymongo from pymongo import MongoClient class mongodbMonitor(object): def mongodb_connect(self,host=None, port=None, user=None, password=None): # try: conn = MongoClient(host, port, serverSelectionTimeoutMS=1000) # conntion timeout 1 sec. if user and password: db_admin = conn["admin"] if not db_admin.authenticate(user,password): pass; conn.server_info() print conn # except : # e = sys.exc_info()[0] # return e, None return 0,conn #data node(1): standalone, replset primary, replset secondary. mongos(2), mongoConfigSrv(3) def get_mongo_role(self, conn): mongo_role = 1 conn.server_info() if (conn.is_mongos): mongo_role = 2 elif ("chunks" in conn.get_database("config").collection_names()): # Role is a config servers? not mongos and has config.chunks collections. it's a config server. mongo_role = 3 return mongo_role def get_mongo_monitor_data(self, conn): mongo_monitor_dict ={} mongo_monitor_dict["mongo_local_alive"] = 1 # mongo local alive metric for all nodes. mongo_role = self.get_mongo_role(conn) if(mongo_role == 1): mongodb_role,serverStatus_dict = self.serverStatus(conn) mongo_monitor_dict.update(serverStatus_dict) repl_status_dict = {} if (mongodb_role == "master" or mongodb_role == "secondary"): repl_status_dict = self.repl_status(conn) mongo_monitor_dict.update(repl_status_dict) else: print "this is standalone node" elif(mongo_role == 2): # mongos shards_dict = self.shard_status(conn) mongo_monitor_dict.update(shards_dict) return mongo_monitor_dict def serverStatus(self,connection): serverStatus = connection.admin.command(pymongo.son_manipulator.SON([('serverStatus', 1)])) mongodb_server_dict = {} # mongodb server status metric for upload to falcon mongo_version = serverStatus["version"] #uptime metric mongodb_server_dict["uptime"] = int(serverStatus["uptime"]) #asserts section metrics mongo_asserts = serverStatus["asserts"] for asserts_key in mongo_asserts.keys(): asserts_key_name = "asserts_" + asserts_key mongodb_server_dict[asserts_key_name] = mongo_asserts[asserts_key] ### "extra_info" section metrics: page_faults. falcon counter type. if serverStatus.has_key("extra_info"): mongodb_server_dict["page_faults"] = serverStatus["extra_info"]["page_faults"] ### "connections" section metrics current_conn = serverStatus["connections"]["current"] available_conn = serverStatus["connections"]["available"] mongodb_server_dict["connections_current"] = current_conn mongodb_server_dict["connections_available"] = available_conn # mongodb connection used percent mongodb_server_dict["connections_used_percent"] = int((current_conn/(current_conn + available_conn)*100)) # total created from mongodb started. COUNTER metric mongodb_server_dict["connections_totalCreated"] = serverStatus["connections"]["totalCreated"] # "globalLock" currentQueue mongodb_server_dict["globalLock_currentQueue_total"] = serverStatus["globalLock"]["currentQueue"]["total"] mongodb_server_dict["globalLock_currentQueue_readers"] = serverStatus["globalLock"]["currentQueue"]["readers"] mongodb_server_dict["globalLock_currentQueue_writers"] = serverStatus["globalLock"]["currentQueue"]["writers"] # "locks" section, Changed in version 3.0 if serverStatus.has_key("locks") and mongo_version >"3.0": locks_dict_keys = serverStatus["locks"].keys() for lock_scope in locks_dict_keys: # Global, Database,Collection,Oplog for lock_metric in serverStatus["locks"][lock_scope]: for lock_type in serverStatus["locks"][lock_scope][lock_metric]: if lock_type == "R": lock_name = "Slock" elif lock_type == "W": lock_name = "Xlock" elif lock_type == "r": lock_name = "ISlock" elif lock_type == "w": lock_name = "IXlock" lock_metric_key = "locks_" + lock_scope + "_" + lock_metric + "_" + lock_name mongodb_server_dict[lock_metric_key] = serverStatus["locks"][lock_scope][lock_metric][lock_type] # "network" section metrics: bytesIn, bytesOut, numRequests; counter type if serverStatus.has_key("network"): for network_metric in serverStatus["network"].keys(): network_metric_key = "network_" + network_metric # network metric key for upload mongodb_server_dict[network_metric_key] = serverStatus["network"][network_metric] ### "opcounters" section metrics: insert, query, update, delete, getmore, command. couter type if serverStatus.has_key("opcounters"): for opcounters_metric in serverStatus["opcounters"].keys(): opcounters_metric_key = "opcounters_" + opcounters_metric mongodb_server_dict[opcounters_metric_key] = serverStatus["opcounters"][opcounters_metric] ### "opcountersRepl" section metrics: insert, query, update, delete, getmore, command. couter type if serverStatus.has_key("opcountersRepl"): for opcountersRepl_metric in serverStatus["opcountersRepl"].keys(): opcountersRepl_metric_key = "opcountersRepl_" + opcountersRepl_metric mongodb_server_dict[opcountersRepl_metric_key] = serverStatus["opcounters"][opcountersRepl_metric] ### "mem" section metrics: if serverStatus.has_key("mem"): for mem_metric in serverStatus["mem"].keys(): mem_metric_key = "mem_" + mem_metric if( mem_metric in ["bits","supported"] ): mongodb_server_dict[mem_metric_key] = serverStatus["mem"][mem_metric] else: mongodb_server_dict[mem_metric_key] = serverStatus["mem"][mem_metric]*1024*1024 ### "dur" section metrics: if serverStatus.has_key("dur"): mongodb_server_dict["dur_journaledBytes"] = serverStatus["dur"]["journaledMB"]*1024*1024 mongodb_server_dict["dur_writeToDataFilesBytes"] = serverStatus["dur"]["writeToDataFilesMB"]*1024*1024 mongodb_server_dict["dur_commitsInWriteLock"] = serverStatus["dur"]["commitsInWriteLock"] ### "repl" section mongodb_role = "" if (serverStatus.has_key("repl") and serverStatus["repl"].has_key("secondary")): if serverStatus["repl"]["ismaster"]: mongodb_role = "master" if serverStatus["repl"]["secondary"]: mongodb_role = "secondary" else: # not Replica sets mode mongodb_role = "standalone" ### "backgroundFlushing" section metrics, only for MMAPv1 if serverStatus.has_key("backgroundFlushing"): for bgFlush_metric in serverStatus["backgroundFlushing"].keys(): if bgFlush_metric != "last_finished": # discard last_finished metric bgFlush_metric_key = "backgroundFlushing_" + bgFlush_metric mongodb_server_dict[bgFlush_metric_key] = serverStatus["backgroundFlushing"][bgFlush_metric] ### cursor from "metrics" section if serverStatus.has_key("metrics") and serverStatus["metrics"].has_key("cursor"): cursor_status = serverStatus["metrics"]["cursor"] mongodb_server_dict["cursor_timedOut"] = cursor_status["timedOut"] mongodb_server_dict["cursor_open_noTimeout"] = cursor_status["open"]["noTimeout"] mongodb_server_dict["cursor_open_pinned"] = cursor_status["open"]["pinned"] mongodb_server_dict["cursor_open_total"] = cursor_status["open"]["total"] ### "wiredTiger" section if serverStatus.has_key("wiredTiger"): serverStatus_wt = serverStatus["wiredTiger"] #cache wt_cache = serverStatus_wt["cache"] mongodb_server_dict["wt_cache_used_total_bytes"] = wt_cache["bytes currently in the cache"] mongodb_server_dict["wt_cache_dirty_bytes"] = wt_cache["tracked dirty bytes in the cache"] mongodb_server_dict["wt_cache_readinto_bytes"] = wt_cache["bytes read into cache"] mongodb_server_dict["wt_cache_writtenfrom_bytes"] = wt_cache["bytes written from cache"] #concurrentTransactions wt_concurrentTransactions = serverStatus_wt["concurrentTransactions"] mongodb_server_dict["wt_concurrentTransactions_write"] = wt_concurrentTransactions["write"]["available"] mongodb_server_dict["wt_concurrentTransactions_read"] = wt_concurrentTransactions["read"]["available"] #"block-manager" section wt_block_manager = serverStatus_wt["block-manager"] mongodb_server_dict["wt_bm_bytes_read"] = wt_block_manager["bytes read"] mongodb_server_dict["wt_bm_bytes_written"] = wt_block_manager["bytes written"] mongodb_server_dict["wt_bm_blocks_read"] = wt_block_manager["blocks read" ] mongodb_server_dict["wt_bm_blocks_written"] = wt_block_manager["blocks written"] ### "rocksdb" engine if serverStatus.has_key("rocksdb"): serverStatus_rocksdb = serverStatus["rocksdb"] mongodb_server_dict["rocksdb_num_immutable_mem_table"] = serverStatus_rocksdb["num-immutable-mem-table"] mongodb_server_dict["rocksdb_mem_table_flush_pending"] = serverStatus_rocksdb["mem-table-flush-pending"] mongodb_server_dict["rocksdb_compaction_pending"] = serverStatus_rocksdb["compaction-pending"] mongodb_server_dict["rocksdb_background_errors"] = serverStatus_rocksdb["background-errors"] mongodb_server_dict["rocksdb_num_entries_active_mem_table"] = serverStatus_rocksdb["num-entries-active-mem-table"] mongodb_server_dict["rocksdb_num_entries_imm_mem_tables"] = serverStatus_rocksdb["num-entries-imm-mem-tables"] mongodb_server_dict["rocksdb_num_snapshots"] = serverStatus_rocksdb["num-snapshots"] mongodb_server_dict["rocksdb_oldest_snapshot_time"] = serverStatus_rocksdb["oldest-snapshot-time"] mongodb_server_dict["rocksdb_num_live_versions"] = serverStatus_rocksdb["num-live-versions"] mongodb_server_dict["rocksdb_total_live_recovery_units"] = serverStatus_rocksdb["total-live-recovery-units"] ### "PerconaFT" engine if serverStatus.has_key("PerconaFT"): serverStatus_PerconaFT = serverStatus["PerconaFT"] mongodb_server_dict["PerconaFT_log_count"] = serverStatus_PerconaFT["log"]["count"] mongodb_server_dict["PerconaFT_log_time"] = serverStatus_PerconaFT["log"]["time"] mongodb_server_dict["PerconaFT_log_bytes"] = serverStatus_PerconaFT["log"]["bytes"] mongodb_server_dict["PerconaFT_fsync_count"] = serverStatus_PerconaFT["fsync"]["count"] mongodb_server_dict["PerconaFT_fsync_time"] = serverStatus_PerconaFT["fsync"]["time"] ### cachetable PerconaFT_cachetable = serverStatus_PerconaFT["cachetable"] mongodb_server_dict["PerconaFT_cachetable_size_current"] = PerconaFT_cachetable["size"]["current"] mongodb_server_dict["PerconaFT_cachetable_size_writing"] = PerconaFT_cachetable["size"]["writing"] mongodb_server_dict["PerconaFT_cachetable_size_limit"] = PerconaFT_cachetable["size"]["limit"] ### PerconaFT checkpoint PerconaFT_checkpoint = serverStatus_PerconaFT["checkpoint"] mongodb_server_dict["PerconaFT_checkpoint_count"] = PerconaFT_checkpoint["count"] mongodb_server_dict["PerconaFT_checkpoint_time"] = PerconaFT_checkpoint["time"] mongodb_server_dict["PerconaFT_checkpoint_write_nonleaf_count"] = PerconaFT_checkpoint["write"]["nonleaf"]["count"] mongodb_server_dict["PerconaFT_checkpoint_write_nonleaf_time"] = PerconaFT_checkpoint["write"]["nonleaf"]["time"] mongodb_server_dict["PerconaFT_checkpoint_write_nonleaf_bytes_compressed"] = PerconaFT_checkpoint["write"]["nonleaf"]["bytes"]["compressed"] mongodb_server_dict["PerconaFT_checkpoint_write_nonleaf_bytes_uncompressed"] = PerconaFT_checkpoint["write"]["nonleaf"]["bytes"]["uncompressed"] mongodb_server_dict["PerconaFT_checkpoint_write_leaf_count"] = PerconaFT_checkpoint["write"]["leaf"]["count"] mongodb_server_dict["PerconaFT_checkpoint_write_leaf_time"] = PerconaFT_checkpoint["write"]["leaf"]["time"] mongodb_server_dict["PerconaFT_checkpoint_write_leaf_bytes_compressed"] = PerconaFT_checkpoint["write"]["leaf"]["bytes"]["compressed"] mongodb_server_dict["PerconaFT_checkpoint_write_leaf_bytes_uncompressed"] = PerconaFT_checkpoint["write"]["leaf"]["bytes"]["uncompressed"] ### serializeTime for serializeTime_item in serverStatus_PerconaFT["serializeTime"]: prefix = "PerconaFT_serializeTime_" + serializeTime_item for serializeTime_key in serverStatus_PerconaFT["serializeTime"][serializeTime_item]: key_name = prefix + "_" + serializeTime_key mongodb_server_dict[key_name] = serverStatus_PerconaFT["serializeTime"][serializeTime_item][serializeTime_key] ### PerconaFT compressionRatio for compressionRatio_item in serverStatus_PerconaFT["compressionRatio"]: key_name = "PerconaFT_compressionRatio_" + compressionRatio_item mongodb_server_dict[key_name] = serverStatus_PerconaFT["compressionRatio"][compressionRatio_item] return (mongodb_role, mongodb_server_dict) def repl_status(self,connection): replStatus = connection.admin.command("replSetGetStatus") print replStatus repl_status_dict = {} # repl set metric dict # myState "1" for PRIMARY , "2" for SECONDARY, "3": repl_status_dict["repl_myState"] = replStatus["myState"] repl_status_members = replStatus["members"] master_optime = 0 # Master oplog ops time myself_optime = 0 # SECONDARY oplog ops time print "开始打印repl_status_members" print repl_status_members print "结束打印repl_status_members" for repl_member in repl_status_members: if repl_member.has_key("self") and repl_member["self"]: repl_status_dict["repl_health"] = repl_member["health"] #repl_status_dict["repl_optime"] = repl_member["optime"].time #print "value of optime ts is:" #print type(repl_member["optime"]) #print type(repl_member["optime"]["ts"]) print repl_member["optime"]["ts"].time repl_status_dict["repl_optime"] = repl_member["optime"]["ts"].time if repl_member.has_key("repl_electionTime"): repl_status_dict["repl_electionTime"] = repl_member["electionTime"].time if repl_member.has_key("repl_configVersion"): repl_status_dict["repl_configVersion"] = repl_member["configVersion"] #myself_optime = repl_member["optime"].time myself_optime = repl_member["optime"]["ts"].time if (replStatus["myState"] == 2 and repl_member["state"] == 1 ): # CONDARY ,get repl lag master_optime = repl_member["optime"]["ts"].time if replStatus["myState"] == 2 : repl_status_dict["repl_lag"] = master_optime - myself_optime ### oplog window hours oplog_collection = connection["local"]["oplog.rs"] oplog_tFirst = oplog_collection.find({},{"ts":1}).sort('$natural',pymongo.ASCENDING).limit(1).next() oplog_tLast = oplog_collection.find({},{"ts":1}).sort('$natural',pymongo.DESCENDING).limit(1).next() oplogrs_collstats = connection["local"].command("collstats", "oplog.rs") window_multiple = 1 ##oplog.rs collections is not full if oplogrs_collstats.has_key("maxSize"): window_multiple = oplogrs_collstats["maxSize"]/(oplogrs_collstats["count"] * oplogrs_collstats["avgObjSize"]) else: window_multiple = oplogrs_collstats["storageSize"]/(oplogrs_collstats["count"] * oplogrs_collstats["avgObjSize"]) #oplog_window .xx hours oplog_window = round((oplog_tLast["ts"].time - oplog_tFirst["ts"].time)/3600.0,2) * window_multiple # full repl_status_dict["repl_oplog_window"] = oplog_window return repl_status_dict # only for mongos node def shard_status(self, conn): config_db = conn["config"] settings_col = config_db["settings"] balancer_doc = settings_col.find_one({'_id':'balancer'}) shards_dict = {} if balancer_doc is None: shards_dict["shards_BalancerState"] = 1 elif balancer_doc["stopped"]: shards_dict["shards_BalancerState"] = 0 else: shards_dict["shards_BalancerState"] = 1 # shards_activeWindow metric,0: without setting, 1:setting # shards_activeWindow_start metric, { "start" : "23:30", "stop" : "6:00" } : 23.30 for 23:30 # shards_activeWindow_stop metric if balancer_doc is None: shards_dict["shards_activeWindow"] = 0 elif balancer_doc.has_key("activeWindow"): shards_dict["shards_activeWindow"] = 1 if balancer_doc["activeWindow"].has_key("start"): window_start = balancer_doc["activeWindow"]["start"] shards_dict["shards_activeWindow_start"] = window_start.replace(":",".") if balancer_doc["activeWindow"].has_key("stop"): window_stop = balancer_doc["activeWindow"]["stop"] shards_dict["shards_activeWindow_stop"] = window_stop.replace(":",".") # shards_chunkSize metric chunksize_doc = settings_col.find_one({"_id" : "chunksize"}) if chunksize_doc is not None: shards_dict["shards_chunkSize"] = chunksize_doc["value"] # shards_isBalancerRunning metric locks_col = config_db["locks"] balancer_lock_doc = locks_col.find_one({'_id':'balancer'}) if balancer_lock_doc is None: print "config.locks collection empty or missing. be sure you are connected to a mongos" shards_dict["shards_isBalancerRunning"] = 0 elif balancer_lock_doc["state"] > 0: shards_dict["shards_isBalancerRunning"] = 1 else: shards_dict["shards_isBalancerRunning"] = 0 # shards_size metric shards_col = config_db["shards"] shards_dict["shards_size"] = shards_col.count() # shards_mongosSize metric mongos_col = config_db["mongos"] shards_dict["shards_mongosSize"] = mongos_col.count() return shards_dict
monodb_monitor.py
#! /bin/env python #-*- coding:utf8 -*- import sys import os import time import datetime import socket import yaml import requests import json from mongodb_server import mongodbMonitor falcon_client = "http://127.0.0.1:1988/v1/push" ts = int(time.time()) # all falcon counter type metrics list mongodb_counter_metric = ["uptime","asserts_msg", "asserts_regular", "asserts_rollovers", "asserts_user", "asserts_warning", "page_faults", "connections_totalCreated", "locks_Global_acquireCount_ISlock", "locks_Global_acquireCount_IXlock", "locks_Global_acquireCount_Slock", "locks_Global_acquireCount_Xlock", "locks_Global_acquireWaitCount_ISlock", "locks_Global_acquireWaitCount_IXlock", "locks_Global_timeAcquiringMicros_ISlock", "locks_Global_timeAcquiringMicros_IXlock", "locks_Database_acquireCount_ISlock", "locks_Database_acquireCount_IXlock", "locks_Database_acquireCount_Slock", "locks_Database_acquireCount_Xlock", "locks_Collection_acquireCount_ISlock", "locks_Collection_acquireCount_IXlock", "locks_Collection_acquireCount_Xlock", "opcounters_command", "opcounters_insert", "opcounters_delete", "opcounters_update", "opcounters_query", "opcounters_getmore", "opcountersRepl_command", "opcountersRepl_insert", "opcountersRepl_delete", "opcountersRepl_update", "opcountersRepl_query", "opcountersRepl_getmore", "network_bytesIn", "network_bytesOut", "network_numRequests", "backgroundFlushing_flushes", "backgroundFlushing_last_ms", "cursor_timedOut", "wt_cache_readinto_bytes", "wt_cache_writtenfrom_bytes", "wt_bm_bytes_read", "wt_bm_bytes_written", "wt_bm_blocks_read", "wt_bm_blocks_written" ] with open('../../cfg.json') as f: data = f.read().replace(' ','') jsonlist = json.loads(data) mongodb_hostname = jsonlist['hostname'] f=open("../conf/mongomon.conf") y = yaml.load(f) f.close() mongodb_items = y["items"] for mongodb_ins in mongodb_items: mongodb_monitor = mongodbMonitor() mongodb_tag = "mongo=" + str(mongodb_ins["port"]) err,conn = mongodb_monitor.mongodb_connect(host="127.0.0.1",port=mongodb_ins["port"], user=mongodb_ins["user"], password=mongodb_ins["password"]) mongodb_upate_list = [] if err != 0: key_item_dict = {"endpoint": mongodb_hostname, "metric": "mongo_local_alive", "tags":mongodb_tag , "timestamp":ts, "value": 0, "step": 60, "counterType": "GAUGE"} mongodb_upate_list.append(key_item_dict) r = requests.post(falcon_client,data=json.dumps(mongodb_upate_list)) continue #The instance is dead. upload the "mongo_alive_local=0" key, then continue. mongodb_dict = mongodb_monitor.get_mongo_monitor_data(conn) mongodb_dict_keys = mongodb_dict.keys() for mongodb_metric in mongodb_dict_keys: if mongodb_metric in mongodb_counter_metric : key_item_dict = {"endpoint": mongodb_hostname, "metric": mongodb_metric, "tags":mongodb_tag , "timestamp":ts, "value": mongodb_dict[mongodb_metric], "step": 60, "counterType": "COUNTER"} else: key_item_dict = {"endpoint": mongodb_hostname, "metric": mongodb_metric, "tags":mongodb_tag , "timestamp":ts, "value": mongodb_dict[mongodb_metric], "step": 60, "counterType": "GAUGE"} mongodb_upate_list.append(key_item_dict) print "开始上报" print json.dumps(mongodb_upate_list) r = requests.post(falcon_client,data=json.dumps(mongodb_upate_list)) print r