Differences

This shows you the differences between two versions of the page.

--- wiki:ai:bcm_servicenow [2026/08/21 19:34] – bgourley
+++ wiki:ai:bcm_servicenow [2026/09/17 12:26] (current) – bgourley
@@ Line 10: / Line 10: @@
 **Auth model confirmed working:** ServiceNow OAuth2 ''client_credentials'' grant against ''/oauth_token.do'', then a Bearer token against ''/api/now/table/incident''.
-----
 ===== Part 1: ServiceNow-Side Prerequisites =====
@@ Line 35: / Line 34: @@
   -H 'Authorization: Bearer PASTE_ACCESS_TOKEN_HERE' \
   -H 'Content-Type: application/json' \
-  -d '{"short_description": "Test incident", "description": "Manual curl test"}' | jq
+  -d '{
+    \"short_description\": \"Test incident from BCM integration testing\",
+    \"description\": \"Manual curl test, ignore/close this\",
+    \"assignment_group\": \"SO-NOC-CLDAI\",
+    \"priority\": \"3\",
+    \"severity\": \"3\",
+    \"company\": \"cdw\",
+    \"configuration_item\": \"CDW-PLACEHOLDER\"
+  }' | jq
 </code>
@@ Line 84: / Line 91: @@
   -X POST "${SN_INSTANCE}/api/now/table/incident" \
   -d "{
-    \"short_description\": \"BCM alert: ${CHECK} on ${NODE}\",
+    \"short_description\": \"Test incident from BCM integration testing\",
-    \"description\": \"Health check/metric ${CHECK} on ${NODE} reported severity ${SEVERITY}, value ${VALUE}\",
+    \"description\": \"Manual curl test, ignore/close this\",
-    \"category\": \"Infrastructure\",
+    \"assignment_group\": \"SO-NOC-CLDAI\",
-    \"urgency\": \"2\",
+    \"priority\": \"3\",
-    \"impact\": \"2\"
+    \"severity\": \"3\",
+    \"company\": \"cdw\",
+    \"configuration_item\": \"CDW-PLACEHOLDER\"
   }" >> /var/log/cmd-servicenow-status.log 2>&1
 </code>
@@ Line 292: / Line 301: @@
 ----
+===== Part 10: Adding Deduplication Logic =====
+<code>
+#!/bin/bash
+# BCM -> ServiceNow incident creation (OAuth2 client_credentials)
+#
+# Config files (both under /cm/local/apps/cmd/scripts/actions/, chmod 600, root-owned):
+#   servicenow-shared.env   - SN_INSTANCE / SN_CLIENT_ID / SN_CLIENT_SECRET
+#                             (same across every client deployment)
+#   servicenow-company.env  - SN_COMPANY_NAME
+#                             (the one value that differs per client deployment)
+SHARED_CONFIG_FILE="/cm/local/apps/cmd/scripts/actions/servicenow-shared.env"
+COMPANY_CONFIG_FILE="/cm/local/apps/cmd/scripts/actions/servicenow-company.env"
+LOG=/var/log/cmd-servicenow-status.log
+log() {
+  echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG"
+}
+# ---------------------------------------------------------------------------
+# Load and validate config
+# ---------------------------------------------------------------------------
+if [ ! -f "$SHARED_CONFIG_FILE" ]; then
+  log "FATAL: shared config file not found at $SHARED_CONFIG_FILE"
+  exit 1
+fi
+if [ ! -f "$COMPANY_CONFIG_FILE" ]; then
+  log "FATAL: company config file not found at $COMPANY_CONFIG_FILE"
+  exit 1
+fi
+# shellcheck disable=SC1090
+source "$SHARED_CONFIG_FILE"
+# shellcheck disable=SC1090
+source "$COMPANY_CONFIG_FILE"
+if [ -z "$SN_INSTANCE" ] || [ -z "$SN_CLIENT_ID" ] || [ -z "$SN_CLIENT_SECRET" ]; then
+  log "FATAL: SN_INSTANCE / SN_CLIENT_ID / SN_CLIENT_SECRET missing from $SHARED_CONFIG_FILE"
+  exit 1
+fi
+if [ -z "$SN_COMPANY_NAME" ]; then
+  log "FATAL: SN_COMPANY_NAME is required but not set in $COMPANY_CONFIG_FILE — refusing to create an unattributed incident"
+  exit 1
+fi
+# ---------------------------------------------------------------------------
+# Alert context from CMDaemon
+# ---------------------------------------------------------------------------
+NODE="${CMD_ENTITY_NAME:-unknown-node}"
+CHECK="${CMD_MEASURABLE_NAME:-unknown-check}"
+SEVERITY="${CMD_SEVERITY:-unknown}"
+VALUE="${CMD_VALUE:-N/A}"
+# ---------------------------------------------------------------------------
+# Step 1: get access token
+# ---------------------------------------------------------------------------
+TOKEN_RESPONSE=$(curl -s -X POST "${SN_INSTANCE}/oauth_token.do" \
+  -H 'Content-Type: application/x-www-form-urlencoded' \
+  --data-urlencode 'grant_type=client_credentials' \
+  --data-urlencode "client_id=${SN_CLIENT_ID}" \
+  --data-urlencode "client_secret=${SN_CLIENT_SECRET}")
+ACCESS_TOKEN=$(echo "${TOKEN_RESPONSE}" | jq -r '.access_token')
+if [ -z "${ACCESS_TOKEN}" ] || [ "${ACCESS_TOKEN}" = "null" ]; then
+  log "Failed to get access token: ${TOKEN_RESPONSE}"
+  exit 1
+fi
+# ---------------------------------------------------------------------------
+# Step 2: resolve Company Name -> sys_id (core_company table)
+# ---------------------------------------------------------------------------
+COMPANY_SYSID=""
+COMPANY_LOOKUP_FAILED=no
+COMPANY_RESPONSE=$(curl -s -w "\n%{http_code}" \
+  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  -H "Content-Type: application/json" \
+  -G "${SN_INSTANCE}/api/now/table/core_company" \
+  --data-urlencode "sysparm_query=name=${SN_COMPANY_NAME}" \
+  --data-urlencode "sysparm_fields=sys_id,name" \
+  --data-urlencode "sysparm_limit=1")
+COMPANY_HTTP_STATUS=$(echo "$COMPANY_RESPONSE" | tail -n1)
+COMPANY_BODY=$(echo "$COMPANY_RESPONSE" | sed '$d')
+if [ "$COMPANY_HTTP_STATUS" = "200" ]; then
+  COMPANY_SYSID=$(echo "$COMPANY_BODY" | jq -r '.result[0].sys_id // empty')
+fi
+if [ -z "$COMPANY_SYSID" ]; then
+  COMPANY_LOOKUP_FAILED=yes
+  log "WARNING: could not resolve company '${SN_COMPANY_NAME}' to a sys_id (HTTP ${COMPANY_HTTP_STATUS}). Incident will be created without a company link and tagged for review."
+fi
+# ---------------------------------------------------------------------------
+# Step 3: build a dedup identity for logging purposes. We no longer write to
+# correlation_id (blocked by an ACL on this instance) — instead we match on
+# short_description containing the check name and node name, scoped by
+# company. NOTE: short_description also contains ${VALUE}, which changes
+# every time the check fires, so we deliberately match on CHECK and NODE as
+# SUBSTRINGS rather than an exact title match — an exact match would treat
+# every differing value as a "new" issue and defeat deduplication entirely.
+# ---------------------------------------------------------------------------
+DEDUP_LABEL="check=${CHECK} node=${NODE} company=${SN_COMPANY_NAME}"
+# ---------------------------------------------------------------------------
+# Step 4: check for an existing OPEN incident matching this check+node
+# (+ company, if resolved). Retries a few times before treating the check
+# as failed.
+# ---------------------------------------------------------------------------
+DEDUP_CHECK_FAILED=no
+EXISTING_SYSID=""
+EXISTING_NUMBER=""
+MATCH_COUNT=0
+DEDUP_QUERY="short_descriptionLIKE[BCM-ALERT]^short_descriptionLIKECheck: ${CHECK} |^short_descriptionLIKENode: ${NODE} |"
+if [ -n "$COMPANY_SYSID" ]; then
+  DEDUP_QUERY="${DEDUP_QUERY}^company=${COMPANY_SYSID}"
+fi
+DEDUP_QUERY="${DEDUP_QUERY}^ORDERBYDESCsys_updated_on"
+for attempt in 1 2 3; do
+  DEDUP_RESPONSE=$(curl -s -w "\n%{http_code}" \
+    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -G "${SN_INSTANCE}/api/now/table/incident" \
+    --data-urlencode "sysparm_query=${DEDUP_QUERY}" \
+    --data-urlencode "sysparm_fields=sys_id,number,sys_updated_on,state" \
+    --data-urlencode "sysparm_display_value=true" \
+    --data-urlencode "sysparm_limit=1")
+  DEDUP_HTTP_STATUS=$(echo "$DEDUP_RESPONSE" | tail -n1)
+  DEDUP_BODY=$(echo "$DEDUP_RESPONSE" | sed '$d')
+  if [ "$DEDUP_HTTP_STATUS" = "200" ]; then
+    MATCH_COUNT=$(echo "$DEDUP_BODY" | jq '.result | length')
+    if [ "$MATCH_COUNT" -gt 0 ]; then
+      TOP_MATCH_STATE=$(echo "$DEDUP_BODY" | jq -r '.result[0].state')
+      if [ "$TOP_MATCH_STATE" = "Resolved" ]; then
+        # Most recent matching incident is already resolved — this is a
+        # fresh recurrence of the issue, not a duplicate. Let it create new.
+        log "Most recent matching incident is Resolved — treating this as a new occurrence, not a duplicate."
+      else
+        EXISTING_SYSID=$(echo "$DEDUP_BODY" | jq -r '.result[0].sys_id')
+        EXISTING_NUMBER=$(echo "$DEDUP_BODY" | jq -r '.result[0].number')
+      fi
+    fi
+    DEDUP_CHECK_FAILED=no
+    break
+  else
+    DEDUP_CHECK_FAILED=yes
+    log "WARNING: dedup check attempt ${attempt}/3 failed (HTTP ${DEDUP_HTTP_STATUS}), retrying..."
+    sleep 2
+  fi
+done
+if [ "$DEDUP_CHECK_FAILED" = "yes" ]; then
+  log "WARNING: dedup check failed after 3 attempts for ${DEDUP_LABEL}. Failing open — will create a new incident tagged as unverified for duplicates."
+fi
+# ---------------------------------------------------------------------------
+# Case A: a genuine duplicate was found -> bump it, don't create a new one.
+# ---------------------------------------------------------------------------
+if [ -n "$EXISTING_SYSID" ]; then
+  NOTE_TEXT="Recurred again at $(date '+%Y-%m-%d %H:%M:%S %Z'). Node: ${NODE}, Check: ${CHECK}, Severity: ${SEVERITY}, Value: ${VALUE}."
+  curl -s -o /dev/null \
+    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -X PATCH "${SN_INSTANCE}/api/now/table/incident/${EXISTING_SYSID}" \
+    -d "{\"work_notes\": \"${NOTE_TEXT}\"}" >> "$LOG" 2>&1
+  log "Duplicate suppressed for ${DEDUP_LABEL} — bumped existing incident ${EXISTING_NUMBER} with a work note instead of creating a new one."
+  exit 0
+fi
+# ---------------------------------------------------------------------------
+# Case B: no duplicate (or dedup check failed open) -> create a new incident.
+# ---------------------------------------------------------------------------
+SHORT_DESC="[BCM-ALERT] Check: ${CHECK} | Node: ${NODE} | Value: ${VALUE}"
+DESC="Health check/metric ${CHECK} on ${NODE}, THIS IS A TEST INCIDENT"
+CONFIG_ITEM="${SN_COMPANY_NAME} - ${NODE}"
+if [ "$DEDUP_CHECK_FAILED" = "yes" ]; then
+  DESC="${DESC} [DEDUP CHECK UNVERIFIED — ServiceNow query failed after retries; this may be a duplicate, please review.]"
+fi
+if [ "$COMPANY_LOOKUP_FAILED" = "yes" ]; then
+  DESC="${DESC} [COMPANY LOOKUP FAILED — no company record matched '${SN_COMPANY_NAME}'; incident created without a company link.]"
+fi
+# Build the JSON payload, only including "company" if we actually resolved one.
+if [ -n "$COMPANY_SYSID" ]; then
+  PAYLOAD=$(jq -n \
+    --arg short_description "$SHORT_DESC" \
+    --arg description "$DESC" \
+    --arg company "$COMPANY_SYSID" \
+    --arg configuration_item "$CONFIG_ITEM" \
+    '{short_description: $short_description, description: $description, assignment_group: "SO-NOC-CLDAI", priority: "3", severity: "3", company: $company, configuration_item: $configuration_item}')
+else
+  PAYLOAD=$(jq -n \
+    --arg short_description "$SHORT_DESC" \
+    --arg description "$DESC" \
+    --arg configuration_item "$CONFIG_ITEM" \
+    '{short_description: $short_description, description: $description, assignment_group: "SO-NOC-CLDAI", priority: "3", severity: "3", configuration_item: $configuration_item}')
+fi
+curl -s -w "\nHTTP_STATUS:%{http_code}\n" \
+  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  -H "Content-Type: application/json" \
+  -X POST "${SN_INSTANCE}/api/now/table/incident" \
+  -d "$PAYLOAD" >> "$LOG" 2>&1
+log "Created new incident for ${DEDUP_LABEL}"
+</code>
 ===== Known Time Sinks to Avoid Next Time =====

Combined Cloud Managed Services

Site Tools

Differences

Page Tools