Combined Cloud Managed Services

User Tools

Site Tools

wiki:ai:bcm_servicenow

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revisionPrevious revision
wiki:ai:bcm_servicenow [2026/08/21 19:38] – [2.1 Script location and content] bgourleywiki:ai:bcm_servicenow [2026/09/17 12:26] (current) – bgourley
Line 301: Line 301:
  
 ---- ----
 +
 +===== Part 10: Adding Deduplication Logic =====
 +
 +<code>
 +#!/bin/bash
 +# BCM -> ServiceNow incident creation (OAuth2 client_credentials)
 +#
 +# Config files (both under /cm/local/apps/cmd/scripts/actions/, chmod 600, root-owned):
 +#   servicenow-shared.env   - SN_INSTANCE / SN_CLIENT_ID / SN_CLIENT_SECRET
 +#                             (same across every client deployment)
 +#   servicenow-company.env  - SN_COMPANY_NAME
 +#                             (the one value that differs per client deployment)
 +
 +SHARED_CONFIG_FILE="/cm/local/apps/cmd/scripts/actions/servicenow-shared.env"
 +COMPANY_CONFIG_FILE="/cm/local/apps/cmd/scripts/actions/servicenow-company.env"
 +LOG=/var/log/cmd-servicenow-status.log
 +
 +log() {
 +  echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG"
 +}
 +
 +# ---------------------------------------------------------------------------
 +# Load and validate config
 +# ---------------------------------------------------------------------------
 +if [ ! -f "$SHARED_CONFIG_FILE" ]; then
 +  log "FATAL: shared config file not found at $SHARED_CONFIG_FILE"
 +  exit 1
 +fi
 +if [ ! -f "$COMPANY_CONFIG_FILE" ]; then
 +  log "FATAL: company config file not found at $COMPANY_CONFIG_FILE"
 +  exit 1
 +fi
 +# shellcheck disable=SC1090
 +source "$SHARED_CONFIG_FILE"
 +# shellcheck disable=SC1090
 +source "$COMPANY_CONFIG_FILE"
 +
 +if [ -z "$SN_INSTANCE" ] || [ -z "$SN_CLIENT_ID" ] || [ -z "$SN_CLIENT_SECRET" ]; then
 +  log "FATAL: SN_INSTANCE / SN_CLIENT_ID / SN_CLIENT_SECRET missing from $SHARED_CONFIG_FILE"
 +  exit 1
 +fi
 +
 +if [ -z "$SN_COMPANY_NAME" ]; then
 +  log "FATAL: SN_COMPANY_NAME is required but not set in $COMPANY_CONFIG_FILE — refusing to create an unattributed incident"
 +  exit 1
 +fi
 +
 +# ---------------------------------------------------------------------------
 +# Alert context from CMDaemon
 +# ---------------------------------------------------------------------------
 +NODE="${CMD_ENTITY_NAME:-unknown-node}"
 +CHECK="${CMD_MEASURABLE_NAME:-unknown-check}"
 +SEVERITY="${CMD_SEVERITY:-unknown}"
 +VALUE="${CMD_VALUE:-N/A}"
 +
 +# ---------------------------------------------------------------------------
 +# Step 1: get access token
 +# ---------------------------------------------------------------------------
 +TOKEN_RESPONSE=$(curl -s -X POST "${SN_INSTANCE}/oauth_token.do" \
 +  -H 'Content-Type: application/x-www-form-urlencoded' \
 +  --data-urlencode 'grant_type=client_credentials' \
 +  --data-urlencode "client_id=${SN_CLIENT_ID}" \
 +  --data-urlencode "client_secret=${SN_CLIENT_SECRET}")
 +
 +ACCESS_TOKEN=$(echo "${TOKEN_RESPONSE}" | jq -r '.access_token')
 +
 +if [ -z "${ACCESS_TOKEN}" ] || [ "${ACCESS_TOKEN}" = "null" ]; then
 +  log "Failed to get access token: ${TOKEN_RESPONSE}"
 +  exit 1
 +fi
 +
 +# ---------------------------------------------------------------------------
 +# Step 2: resolve Company Name -> sys_id (core_company table)
 +# ---------------------------------------------------------------------------
 +COMPANY_SYSID=""
 +COMPANY_LOOKUP_FAILED=no
 +
 +COMPANY_RESPONSE=$(curl -s -w "\n%{http_code}" \
 +  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
 +  -H "Content-Type: application/json" \
 +  -G "${SN_INSTANCE}/api/now/table/core_company" \
 +  --data-urlencode "sysparm_query=name=${SN_COMPANY_NAME}" \
 +  --data-urlencode "sysparm_fields=sys_id,name" \
 +  --data-urlencode "sysparm_limit=1")
 +
 +COMPANY_HTTP_STATUS=$(echo "$COMPANY_RESPONSE" | tail -n1)
 +COMPANY_BODY=$(echo "$COMPANY_RESPONSE" | sed '$d')
 +
 +if [ "$COMPANY_HTTP_STATUS" = "200" ]; then
 +  COMPANY_SYSID=$(echo "$COMPANY_BODY" | jq -r '.result[0].sys_id // empty')
 +fi
 +
 +if [ -z "$COMPANY_SYSID" ]; then
 +  COMPANY_LOOKUP_FAILED=yes
 +  log "WARNING: could not resolve company '${SN_COMPANY_NAME}' to a sys_id (HTTP ${COMPANY_HTTP_STATUS}). Incident will be created without a company link and tagged for review."
 +fi
 +
 +# ---------------------------------------------------------------------------
 +# Step 3: build a dedup identity for logging purposes. We no longer write to
 +# correlation_id (blocked by an ACL on this instance) — instead we match on
 +# short_description containing the check name and node name, scoped by
 +# company. NOTE: short_description also contains ${VALUE}, which changes
 +# every time the check fires, so we deliberately match on CHECK and NODE as
 +# SUBSTRINGS rather than an exact title match — an exact match would treat
 +# every differing value as a "new" issue and defeat deduplication entirely.
 +# ---------------------------------------------------------------------------
 +DEDUP_LABEL="check=${CHECK} node=${NODE} company=${SN_COMPANY_NAME}"
 +
 +# ---------------------------------------------------------------------------
 +# Step 4: check for an existing OPEN incident matching this check+node
 +# (+ company, if resolved). Retries a few times before treating the check
 +# as failed.
 +# ---------------------------------------------------------------------------
 +DEDUP_CHECK_FAILED=no
 +EXISTING_SYSID=""
 +EXISTING_NUMBER=""
 +MATCH_COUNT=0
 +
 +DEDUP_QUERY="short_descriptionLIKE[BCM-ALERT]^short_descriptionLIKECheck: ${CHECK} |^short_descriptionLIKENode: ${NODE} |"
 +if [ -n "$COMPANY_SYSID" ]; then
 +  DEDUP_QUERY="${DEDUP_QUERY}^company=${COMPANY_SYSID}"
 +fi
 +DEDUP_QUERY="${DEDUP_QUERY}^ORDERBYDESCsys_updated_on"
 +
 +for attempt in 1 2 3; do
 +  DEDUP_RESPONSE=$(curl -s -w "\n%{http_code}" \
 +    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
 +    -H "Content-Type: application/json" \
 +    -G "${SN_INSTANCE}/api/now/table/incident" \
 +    --data-urlencode "sysparm_query=${DEDUP_QUERY}" \
 +    --data-urlencode "sysparm_fields=sys_id,number,sys_updated_on,state" \
 +    --data-urlencode "sysparm_display_value=true" \
 +    --data-urlencode "sysparm_limit=1")
 +
 +  DEDUP_HTTP_STATUS=$(echo "$DEDUP_RESPONSE" | tail -n1)
 +  DEDUP_BODY=$(echo "$DEDUP_RESPONSE" | sed '$d')
 +
 +  if [ "$DEDUP_HTTP_STATUS" = "200" ]; then
 +    MATCH_COUNT=$(echo "$DEDUP_BODY" | jq '.result | length')
 +    if [ "$MATCH_COUNT" -gt 0 ]; then
 +      TOP_MATCH_STATE=$(echo "$DEDUP_BODY" | jq -r '.result[0].state')
 +      if [ "$TOP_MATCH_STATE" = "Resolved" ]; then
 +        # Most recent matching incident is already resolved — this is a
 +        # fresh recurrence of the issue, not a duplicate. Let it create new.
 +        log "Most recent matching incident is Resolved — treating this as a new occurrence, not a duplicate."
 +      else
 +        EXISTING_SYSID=$(echo "$DEDUP_BODY" | jq -r '.result[0].sys_id')
 +        EXISTING_NUMBER=$(echo "$DEDUP_BODY" | jq -r '.result[0].number')
 +      fi
 +    fi
 +    DEDUP_CHECK_FAILED=no
 +    break
 +  else
 +    DEDUP_CHECK_FAILED=yes
 +    log "WARNING: dedup check attempt ${attempt}/3 failed (HTTP ${DEDUP_HTTP_STATUS}), retrying..."
 +    sleep 2
 +  fi
 +done
 +
 +if [ "$DEDUP_CHECK_FAILED" = "yes" ]; then
 +  log "WARNING: dedup check failed after 3 attempts for ${DEDUP_LABEL}. Failing open — will create a new incident tagged as unverified for duplicates."
 +fi
 +
 +# ---------------------------------------------------------------------------
 +# Case A: a genuine duplicate was found -> bump it, don't create a new one.
 +# ---------------------------------------------------------------------------
 +if [ -n "$EXISTING_SYSID" ]; then
 +  NOTE_TEXT="Recurred again at $(date '+%Y-%m-%d %H:%M:%S %Z'). Node: ${NODE}, Check: ${CHECK}, Severity: ${SEVERITY}, Value: ${VALUE}."
 +
 +  curl -s -o /dev/null \
 +    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
 +    -H "Content-Type: application/json" \
 +    -X PATCH "${SN_INSTANCE}/api/now/table/incident/${EXISTING_SYSID}" \
 +    -d "{\"work_notes\": \"${NOTE_TEXT}\"}" >> "$LOG" 2>&1
 +
 +  log "Duplicate suppressed for ${DEDUP_LABEL} — bumped existing incident ${EXISTING_NUMBER} with a work note instead of creating a new one."
 +  exit 0
 +fi
 +
 +# ---------------------------------------------------------------------------
 +# Case B: no duplicate (or dedup check failed open) -> create a new incident.
 +# ---------------------------------------------------------------------------
 +SHORT_DESC="[BCM-ALERT] Check: ${CHECK} | Node: ${NODE} | Value: ${VALUE}"
 +DESC="Health check/metric ${CHECK} on ${NODE}, THIS IS A TEST INCIDENT"
 +CONFIG_ITEM="${SN_COMPANY_NAME} - ${NODE}"
 +
 +if [ "$DEDUP_CHECK_FAILED" = "yes" ]; then
 +  DESC="${DESC} [DEDUP CHECK UNVERIFIED — ServiceNow query failed after retries; this may be a duplicate, please review.]"
 +fi
 +if [ "$COMPANY_LOOKUP_FAILED" = "yes" ]; then
 +  DESC="${DESC} [COMPANY LOOKUP FAILED — no company record matched '${SN_COMPANY_NAME}'; incident created without a company link.]"
 +fi
 +
 +# Build the JSON payload, only including "company" if we actually resolved one.
 +if [ -n "$COMPANY_SYSID" ]; then
 +  PAYLOAD=$(jq -n \
 +    --arg short_description "$SHORT_DESC" \
 +    --arg description "$DESC" \
 +    --arg company "$COMPANY_SYSID" \
 +    --arg configuration_item "$CONFIG_ITEM" \
 +    '{short_description: $short_description, description: $description, assignment_group: "SO-NOC-CLDAI", priority: "3", severity: "3", company: $company, configuration_item: $configuration_item}')
 +else
 +  PAYLOAD=$(jq -n \
 +    --arg short_description "$SHORT_DESC" \
 +    --arg description "$DESC" \
 +    --arg configuration_item "$CONFIG_ITEM" \
 +    '{short_description: $short_description, description: $description, assignment_group: "SO-NOC-CLDAI", priority: "3", severity: "3", configuration_item: $configuration_item}')
 +fi
 +
 +curl -s -w "\nHTTP_STATUS:%{http_code}\n" \
 +  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
 +  -H "Content-Type: application/json" \
 +  -X POST "${SN_INSTANCE}/api/now/table/incident" \
 +  -d "$PAYLOAD" >> "$LOG" 2>&1
 +
 +log "Created new incident for ${DEDUP_LABEL}"
 +
 +</code>
  
 ===== Known Time Sinks to Avoid Next Time ===== ===== Known Time Sinks to Avoid Next Time =====
wiki/ai/bcm_servicenow.1787341101.txt.gz · Last modified: by bgourley

Page Tools

Trace: • bcm_servicenow
Donate Powered by PHP Valid HTML5 Valid CSS Driven by DokuWiki

Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com) IP:10.0.142.190