{ "schemaVersion": 1, "updated": "2026-09-10", "title": "The AI security framework", "status": "An editorial guide to capabilities, not a certification or product evaluation.", "pillars": [ { "id": "employees", "number": "01", "name": "Employees", "label": "The AI people use", "icon": "person", "color": "#90ddd2", "question": "Which AI tools are people using—and what are they sharing?", "risk": "Sensitive information can leave through an unapproved tool. A convincing message can trick someone into handing over access.", "promise": "Make safe use easier.", "summary": "Discover AI use, protect information, and catch deceptive messages.", "analogy": "Give people an approved front door and a clear way to check who is on the other side.", "controls": [ { "title": "Know what is in use", "text": "Identify AI services, connected apps, the people using them, and the data they can access. Offer an approved route for common tasks." }, { "title": "Protect the moment of sharing", "text": "Warn or block sensitive transfers where policy requires it. Check app permissions and detect suspicious messages and account activity." }, { "title": "Verify consequential requests", "text": "Confirm payment, account-change, or sensitive-data requests through a separate trusted channel. Do not share passwords or sign-in codes. Give people a simple way to report concerns." } ], "technologies": [ "AI usage discovery", "Data loss prevention", "Email and messaging security", "Identity protection" ], "proof": "Show one sensitive-data test from warning to resolution, and list the apps and devices that are outside coverage.", "owner": "Workplace IT + security + privacy", "limit": "Discovery can miss personal accounts or unmanaged devices. Polished text alone does not prove AI involvement; blocking prompts cannot eliminate deception.", "sourceIds": [ "nist-ai-rmf", "nist-csf-2", "fbi-bec", "ftc-verification-codes" ] }, { "id": "agents", "number": "02", "name": "Agents", "label": "The AI that takes action", "icon": "boundary", "color": "#d4e99a", "question": "What can an agent do, and who gave it permission?", "risk": "An agent can mistake a hostile document for an instruction and use its permitted tools in a harmful way. Excessive access can make the damage worse.", "promise": "Put boundaries around action.", "summary": "Inventory agents, limit their permissions, and control consequential actions.", "analogy": "Give a digital assistant a job badge with access to specific rooms. Check what it does inside them, too.", "controls": [ { "title": "Name the owner and the job", "text": "Record each agent, its purpose, identity, tools, data access, and accountable owner. Include agents supplied by vendors." }, { "title": "Enforce limits outside the model", "text": "Restrict tools, destinations, credentials, and permissions to the task. Treat instructions found in documents or websites as untrusted input." }, { "title": "Check and stop real actions", "text": "Require approval for high-impact actions. Record tool calls and outcomes. Test whether revoking access actually stops a running agent." } ], "technologies": [ "Agent inventory", "Identity and access controls", "Tool authorization", "Agent activity monitoring" ], "proof": "Test attempts to reach an unapproved system, show the enforced block, and explain the test’s limits. Demonstrate how you revoke the agent’s access.", "owner": "AI platform + application owners + security", "limit": "A prompt filter or a written rule is not an access boundary. These controls apply to agents you operate or procure; they do not govern an attacker’s agents.", "sourceIds": [ "owasp-llm", "owasp-agentic", "cisa-secure-ai" ] }, { "id": "cloud", "number": "03", "name": "Cloud", "label": "The systems AI can reach", "icon": "key", "color": "#efb2c6", "question": "If one account is compromised, how far can it go?", "risk": "A stolen key or an exposed service can give an intruder access to connected systems and sensitive records.", "promise": "Limit access. Spot misuse.", "summary": "Reduce exposed paths and detect unusual behavior across identities and systems.", "analogy": "Lock individual rooms, notice unusual movement, and keep a record of who opened which door.", "controls": [ { "title": "Reduce paths into the environment", "text": "Map exposed services and permissions. Remove unnecessary access, protect secrets, and separate sensitive systems." }, { "title": "Watch behavior across services", "text": "Connect identity, cloud, and data-access events. Investigate unusual sign-ins, privilege changes, and unexpected data movement." }, { "title": "Contain and recover", "text": "Be able to revoke credentials, isolate workloads, and preserve evidence. Test restoration and recovery for essential services." } ], "technologies": [ "Cloud posture management", "Cloud detection and response", "Secrets management", "Identity threat protection" ], "proof": "Trace a test credential from initial use to detection and revocation. Explain which connected systems remain reachable.", "owner": "Cloud platform + identity + security operations", "limit": "Good configuration does not catch every misuse of legitimate access. Behavior monitoring depends on available logs, reliable detection, and timely response.", "sourceIds": [ "nist-csf-2", "cisa-secure-ai" ] }, { "id": "software", "number": "04", "name": "Software", "label": "The code your organization runs", "icon": "file", "color": "#eac18c", "question": "Can we find and fix exploitable flaws before they are used?", "risk": "A flaw in an application or a third-party component can become an entry point. A rushed fix can introduce a new problem.", "promise": "Find, fix, and verify.", "summary": "Use AI to help review code and propose repairs, with validation before release.", "analogy": "Use an assistant to inspect and repair the building, then have the repair checked before people rely on it.", "controls": [ { "title": "Find and prioritize", "text": "Inventory software and dependencies. Combine security testing with AI-assisted review; prioritize flaws by exposure, exploitability, and business impact." }, { "title": "Propose and review a repair", "text": "Use AI where useful to explain findings and draft changes. An accountable reviewer checks the security reasoning and the effect on the application." }, { "title": "Test, release, and verify", "text": "Test that the flaw is addressed and essential behavior still works. Use an approved deployment process, a rollback plan, and checks on the running system." } ], "technologies": [ "Code and dependency scanning", "AI-assisted code review", "Vulnerability prioritization", "Patch validation and deployment" ], "proof": "Take one real finding through a tested fix into the running application. Show the evidence that the flaw is resolved and how to roll back.", "owner": "Engineering + application security + service owners", "limit": "AI can miss flaws, invent findings, or propose insecure repairs. No tool finds and patches every vulnerability; a suggested change is not a deployed fix.", "sourceIds": [ "nist-ssdf", "cisa-secure-ai" ] }, { "id": "operations", "number": "05", "name": "Security operations", "label": "The AI helping defenders", "icon": "scan", "color": "#c3b7ee", "question": "Can defenders understand and interrupt an incident in time?", "risk": "Clues can sit across separate tools while an intrusion continues. An automated response can also disrupt legitimate work.", "promise": "Turn signals into decisions.", "summary": "Help analysts connect evidence, investigate, and act with clear authority.", "analogy": "Give the response team a shared incident room, with evidence on the wall and a clear person in charge.", "controls": [ { "title": "Bring the evidence together", "text": "Collect relevant identity, cloud, application, and agent events with timestamps. Protect log integrity and limit access to sensitive content." }, { "title": "Assist the investigation", "text": "Use AI to organize evidence, suggest hypotheses, and draft detections. Keep links to original events so analysts can check the conclusions." }, { "title": "Bound the response", "text": "Define which actions can be automated and which require approval. Test playbooks, keep action records, and rehearse containment and recovery." } ], "technologies": [ "Security data platform", "AI investigation workbench", "Detection engineering", "Response orchestration" ], "proof": "Run a realistic exercise. Measure time to a correct decision and containment, alongside false alarms and unintended disruption.", "owner": "Security operations + incident response", "limit": "More data or a faster summary does not guarantee a correct conclusion. Security AI needs its own access limits, evaluation, and oversight.", "sourceIds": [ "nist-csf-2", "nist-ai-rmf", "owasp-llm" ] } ], "governance": { "title": "Governance is the foundation.", "summary": "Know what you have. Name an owner. Set the rules. Keep evidence that they work.", "items": [ { "title": "Discover & assign", "text": "Maintain an inventory, purpose, supplier, data access, and accountable owner." }, { "title": "Set policy", "text": "Define allowed uses, sensitive data, approval thresholds, and requirements for vendors." }, { "title": "Check the evidence", "text": "Test controls and record exceptions, incidents, coverage gaps, and remediation." }, { "title": "Review & improve", "text": "Reassess as tools, permissions, risks, and organizational obligations change." } ], "sourceIds": [ "nist-ai-rmf", "nist-csf-2" ] }, "technologyDefinitions": { "AI usage discovery": "Find which AI services and connected apps people use.", "Data loss prevention": "Detect or restrict sharing of specified sensitive information.", "Email and messaging security": "Identify suspicious messages and attempts to impersonate trusted people.", "Identity protection": "Detect suspicious account access and reduce account takeover risk.", "Agent inventory": "A record of agents, their owners, purposes, tools, and permissions.", "Identity and access controls": "Decide who or what may reach a resource, and what actions are allowed.", "Tool authorization": "Check an agent’s permission before a tool performs an action.", "Agent activity monitoring": "Record and inspect what agents ask their tools to do and what happens.", "Cloud posture management": "Check cloud settings, exposure, and permissions for risky configurations.", "Cloud detection and response": "Find suspicious activity in cloud systems and help contain it.", "Secrets management": "Protect, limit, and rotate passwords, keys, and other access credentials.", "Identity threat protection": "Connect account activity to signs of compromise or misuse.", "Code and dependency scanning": "Check application code and third-party components for known or suspected flaws.", "AI-assisted code review": "Use AI to examine code, explain possible problems, and propose changes.", "Vulnerability prioritization": "Decide which flaws to address first based on their practical risk.", "Patch validation and deployment": "Check a repair, release it safely, and verify it is running.", "Security data platform": "Bring security events together so defenders can search and connect them.", "AI investigation workbench": "An environment where analysts use AI to examine evidence and develop hypotheses.", "Detection engineering": "Create and test rules or models that identify suspicious behavior.", "Response orchestration": "Coordinate response tools and actions with defined approvals." }, "scenarios": [ { "id": "outside", "label": "An attacker uses AI", "title": "Protect what the attacker can reach.", "intro": "In the 2025 AI-orchestrated espionage case, attackers used AI to perform much of the tactical work. The organizations being targeted did not control that AI.", "incidentId": "gtg-1002-2025", "stages": [ { "action": "Look for a way in", "defense": "Find exposed systems and fix reachable flaws.", "pillarId": "software" }, { "action": "Use access to move further", "defense": "Limit permissions and protect credentials.", "pillarId": "cloud" }, { "action": "Reach and copy sensitive data", "defense": "Spot unusual access and interrupt the intrusion.", "pillarId": "operations" } ], "limit": "This is an illustrative control map, not a reconstruction of every step or evidence that these controls were absent." }, { "id": "inside", "label": "Your agent is redirected", "title": "Check permission where AI becomes action.", "intro": "In the evaluation-pipeline attack reported as GTG-50020, malicious instructions caused a defender’s AI workflow to release access keys.", "incidentId": "gtg-50020", "stages": [ { "action": "An agent reads hostile material", "defense": "Treat outside text as data, not trusted authority.", "pillarId": "agents" }, { "action": "A tool is asked to expose a secret", "defense": "Check tool permissions; keep secrets outside untrusted tasks.", "pillarId": "agents" }, { "action": "A leaked key could be reused", "defense": "Limit its reach, detect misuse, and revoke access.", "pillarId": "cloud" } ], "limit": "Prompt screening alone cannot guarantee safety. The third step describes a risk to contain, not an additional reported outcome." }, { "id": "people", "label": "A person trusts a fake tool", "title": "Make the trustworthy route easy to find.", "intro": "In GTG-50021, a counterfeit Claude service delivered a different model and installed software designed to steal credentials.", "incidentId": "gtg-50021", "stages": [ { "action": "A person looks for AI access", "defense": "Provide approved tools and a clear way to request access.", "pillarId": "employees" }, { "action": "An untrusted app asks for access", "defense": "Check the supplier, downloads, and requested permissions.", "pillarId": "employees" }, { "action": "An account may be put at risk", "defense": "Detect unusual account use and remove compromised access.", "pillarId": "cloud" } ], "limit": "The case concerns an AI-themed lure. The source does not establish that AI carried out the attack or that all users lost credentials." } ], "sources": [ { "id": "nist-ai-rmf", "title": "AI Risk Management Framework 1.0: Core", "publisher": "NIST", "url": "https://airc.nist.gov/airmf-resources/airmf/5-sec-core/", "kind": "guidance", "verified": "2026-09-10", "claims": [ "GOVERN 1.6 calls for an inventory of AI systems; GOVERN 1.5 calls for ongoing monitoring, periodic review and clear responsibilities.", "Governance spans the framework's Map, Measure and Manage functions; these outcomes are not a certification checklist." ] }, { "id": "nist-csf-2", "title": "The NIST Cybersecurity Framework 2.0", "publisher": "NIST", "url": "https://doi.org/10.6028/NIST.CSWP.29", "kind": "guidance", "verified": "2026-09-10", "claims": [ "PR.AA addresses identity, authentication and access; PR.AA-05 specifies least privilege and separation of duties.", "DE.CM calls for monitoring networks, technology usage, service providers and runtime environments for adverse events.", "Its Govern, Identify, Protect, Detect, Respond and Recover functions address the whole risk management cycle." ] }, { "id": "owasp-llm", "title": "LLM06:2025 Excessive Agency", "publisher": "OWASP Gen AI Security Project", "url": "https://genai.owasp.org/llmrisk/llm062025-excessive-agency/", "kind": "guidance", "verified": "2026-09-10", "claims": [ "Reduce unnecessary tools, permissions and autonomy; require review of high-impact actions and enforce authorization in downstream systems.", "Logging, monitoring and rate limits can limit damage; OWASP distinguishes these from preventing excessive agency." ] }, { "id": "owasp-agentic", "title": "OWASP Top 10 for Agentic Applications 2026", "publisher": "OWASP Gen AI Security Project", "url": "https://genai.owasp.org/download/52117/?tmstv=1765059207", "kind": "guidance", "verified": "2026-09-10", "claims": [ "ASI03 recommends per-agent identities and task-scoped, time-bound permissions; ASI04 recommends component inventories and provenance.", "ASI08 recommends isolation, network boundaries and checks before consequential tool use; ASI10 includes monitoring and rapid containment." ] }, { "id": "nist-ssdf", "title": "Secure Software Development Framework 1.1", "publisher": "NIST", "url": "https://csrc.nist.gov/pubs/sp/800/218/final", "documentUrl": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218.pdf", "kind": "guidance", "verified": "2026-09-10", "claims": [ "PW.7 and PW.8 cover code review, analysis and testing; RV.2 calls for risk-based vulnerability assessment and remediation.", "RV.3 addresses root causes and similar flaws. These practices aim to reduce vulnerabilities, not guarantee their elimination.", "NIST lists version 1.1 as Final and version 1.2 as Draft. The framework does not establish that AI-generated patches are safe without validation." ] }, { "id": "cisa-secure-ai", "title": "Guidelines for Secure AI System Development", "publisher": "UK NCSC, CISA and international partners", "url": "https://www.ncsc.gov.uk/collection/guidelines-secure-ai-system-development", "kind": "guidance", "verified": "2026-09-10", "claims": [ "The joint guidance covers secure design, development, deployment, and operation and maintenance.", "It recommends infrastructure protection, incident management, logging and monitoring, and secure update management throughout the AI lifecycle." ], "sections": [ { "label": "Staff awareness and design choices", "url": "https://www.ncsc.gov.uk/collection/guidelines-secure-ai-system-development/guidelines/secure-design" }, { "label": "Assets, access and sensitive logs", "url": "https://www.ncsc.gov.uk/collection/guidelines-secure-ai-system-development/guidelines/secure-development" }, { "label": "Infrastructure and response", "url": "https://www.ncsc.gov.uk/collection/guidelines-secure-ai-system-development/guidelines/secure-deployment" }, { "label": "Monitoring and updates", "url": "https://www.ncsc.gov.uk/collection/guidelines-secure-ai-system-development/guidelines/secure-operation-maintenance" } ] }, { "id": "abnormal-ai-governance", "title": "AI Governance", "publisher": "Abnormal AI", "url": "https://abnormal.ai/platform/ai-governance", "kind": "vendor-description", "verified": "2026-09-10", "claims": [ "Abnormal describes AI usage discovery from email, identity, SaaS, provider APIs and optional browser signals, risk scoring, conversation exposure alerts and governance workflows.", "The page includes a roadmap disclaimer: descriptions of future functionality are not availability or performance commitments. Public marketing descriptions alone do not verify coverage or deployment readiness." ] }, { "id": "abnormal-platform", "title": "The Behavioral Security Platform", "publisher": "Abnormal AI", "url": "https://abnormal.ai/platform/overview", "kind": "vendor-description", "verified": "2026-09-10", "claims": [ "The public product navigation describes email and messaging security, account takeover protection, identity threat protection, AI governance, and AI-assisted mailbox triage.", "This is a vendor description, not independent evidence of effectiveness against any incident in the library." ] }, { "id": "fbi-bec", "title": "Business Email Compromise", "publisher": "FBI", "url": "https://www.fbi.gov/how-we-can-help-you/common-frauds-and-scams/business-email-compromise", "kind": "guidance", "verified": "2026-09-10", "claims": [ "Verify payment or account-change requests through an independently checked phone number or in person, rather than relying on the contact details supplied in the message." ] }, { "id": "ftc-verification-codes", "title": "Protect verification codes from impersonation scams", "publisher": "Federal Trade Commission", "url": "https://consumer.ftc.gov/consumer-alerts/2024/06/got-call-about-fraud-activity-your-bank-account-it-could-be-scammer", "kind": "guidance", "verified": "2026-09-10", "claims": [ "Do not share account verification codes with callers. Check purported fraud warnings through a trusted contact route." ] } ], "incidentDefenses": [ { "incidentId": "hugging-face-2026", "framing": "The reported test escape connects two responsibilities: contain your own AI experiments and protect the systems they might reach.", "relevance": "direct", "connections": [ { "pillarId": "agents", "title": "Enforce the test boundary", "why": "The organization running a test can restrict its agents to approved systems and connections, with an independent way to stop their actions.", "limitation": "An instruction to stay inside the test is insufficient; the actual network and tool restrictions need testing." }, { "pillarId": "cloud", "title": "Limit what a stolen key opens", "why": "Separate research access from production access, restrict account permissions, and revoke exposed credentials to reduce how far an intrusion can spread.", "limitation": "The case involved several systems and routes of access. A single account restriction would not address every route." }, { "pillarId": "software", "title": "Find and repair reachable flaws", "why": "Review the software exposed to outside users, fix exploitable weaknesses, and verify the deployed fix against the original problem.", "limitation": "Testing can miss vulnerabilities, and a correct patch does not remove credentials an intruder already obtained." } ] }, { "incidentId": "claude-opus-46-third-party-access", "framing": "Anthropic reports that the model tried to stop, but the test's stopping mechanism failed before outside access occurred.", "relevance": "direct", "connections": [ { "pillarId": "agents", "title": "Test the stop mechanism", "why": "The test operator can verify that a stop request actually cuts off tools and network access, even when the task has gone wrong.", "limitation": "This protects an organization's own agents. A potential victim cannot operate an outside attacker's stop mechanism." }, { "pillarId": "cloud", "title": "Contain account access", "why": "For the affected system, narrow permissions and monitoring of administrator activity can limit or reveal unexpected access and settings changes.", "limitation": "These are relevant control categories; the published account does not establish how each was configured at the affected organization." } ] }, { "incidentId": "claude-opus-47-mistaken-company", "framing": "In the reported evaluation, a matching company name led the AI to a real business outside its authorized task.", "relevance": "direct", "connections": [ { "pillarId": "agents", "title": "Specify authorized destinations", "why": "The test operator can define the exact systems an agent may reach and enforce that list outside the model's own judgment.", "limitation": "Names alone are ambiguous. Restrictions must cover the actual network destinations and tools the agent can use." }, { "pillarId": "cloud", "title": "Separate reading from changing", "why": "On the business side, accounts that only need to read records should not also be able to alter them; unusual database actions need review.", "limitation": "Read-only access can still expose private information. Preventing changes does not by itself prevent theft." } ] }, { "incidentId": "claude-mythos-pypi-package", "framing": "Anthropic reports that a test published harmful software into a shared library, and leaked credentials then opened another system.", "relevance": "direct", "connections": [ { "pillarId": "agents", "title": "Control public publishing", "why": "An evaluation should not give an agent permission to publish software publicly unless that action is explicitly authorized and independently checked.", "limitation": "This applies to the test operator. People receiving a package cannot control its author's agent permissions." }, { "pillarId": "software", "title": "Isolate untrusted packages", "why": "Run unfamiliar software in a separate environment that cannot read real credentials or reach sensitive systems, including when analyzing suspicious packages.", "limitation": "Package analysis may need to execute code. Its isolation and network limits must hold during that execution." }, { "pillarId": "cloud", "title": "Reduce the reach of credentials", "why": "Keep database keys out of package-running environments where possible; limit their permissions and replace them when exposure is discovered.", "limitation": "Replacing a key stops its future use but does not retrieve information already copied." } ] }, { "incidentId": "claude-research-model-network-scan", "framing": "The reported test expanded into a wide search when its intended target disappeared, then reached an unrelated company's application.", "relevance": "direct", "connections": [ { "pillarId": "agents", "title": "Stop when the scope breaks", "why": "If an approved test target is unavailable, the test operator can require the run to stop rather than let the agent search other systems.", "limitation": "A written rule needs an enforced connection boundary; later self-correction cannot undo earlier access." }, { "pillarId": "operations", "title": "Connect suspicious activity", "why": "For a target organization, linking unusual application requests, file access, and unexpected running programs can help investigators recognize an intrusion.", "limitation": "Detection depends on available records and timely response. Scanning alone is not proof that a system was breached." } ] }, { "incidentId": "gtg-20006", "framing": "The reported espionage campaign includes a linked operation that lured travelers through fake update or sign-in prompts.", "relevance": "direct", "connections": [ { "pillarId": "employees", "title": "Protect the sign-in moment", "why": "Managed software installation, trusted update channels, and phishing-resistant sign-in can help protect staff when a convincing prompt asks them to install something or sign in.", "limitation": "The traveler example comes from a linked subcampaign. It is not an established entry route for every reported target." }, { "pillarId": "operations", "title": "Investigate behavior across systems", "why": "Security teams can connect suspicious device activity with account use and access to sensitive records, then contain the affected device or account.", "limitation": "AI-assisted analysis still needs reliable records and validation; it cannot reveal activity on systems the team cannot observe." } ] }, { "incidentId": "gtg-50014", "framing": "The report describes information theft at a vendor with consequences for organizations further down the supply chain.", "relevance": "direct", "connections": [ { "pillarId": "cloud", "title": "Limit supplier access and data", "why": "Give each supplier only the information and connections its job requires, and make that access easy to suspend during an incident.", "limitation": "Local access restrictions cannot erase copies already held by a supplier; data sharing and retention decisions also matter." }, { "pillarId": "operations", "title": "Prepare a supplier response", "why": "Know who can coordinate with the vendor, identify affected data, revoke shared access, and inform people whose information may be exposed.", "limitation": "Response depends partly on the vendor's visibility and cooperation. Monitoring one organization does not provide a complete view of another." } ] }, { "incidentId": "gtg-10007", "framing": "The operators reportedly used their own AI agents for vulnerability research and intrusions; targets must defend the systems those agents approach.", "relevance": "direct", "connections": [ { "pillarId": "software", "title": "Reduce exposed weaknesses", "why": "Identify software reachable from outside, prioritize exploitable flaws, and test and deploy fixes; AI can assist with reviewing code and proposing repairs.", "limitation": "The summary does not identify every exploited flaw. No scanner or AI reviewer establishes that an application is vulnerability-free." }, { "pillarId": "cloud", "title": "Limit access to sensitive records", "why": "Separate accounts and systems so that access to one application does not automatically grant access to student or citizen records elsewhere.", "limitation": "An attacker using valid credentials can resemble legitimate use. Permission limits need monitoring and a response process." } ] }, { "incidentId": "gtg-50021", "framing": "Here AI was the advertised product and lure: the reported reseller delivered a different model and credential-stealing software.", "relevance": "direct", "connections": [ { "pillarId": "employees", "title": "Make trusted AI access easy", "why": "Provide approved AI services through verified channels, and review unfamiliar tools before employees install them or connect work accounts.", "limitation": "An approved-tools list alone cannot stop a harmful download; installation and account controls must cover the devices people use." }, { "pillarId": "operations", "title": "Respond to stolen credentials", "why": "Investigate unexpected software and account activity, remove the harmful software, and revoke exposed passwords, keys, and active sessions as appropriate.", "limitation": "Changing a password alone may leave active sessions or installed malware in place. The affected access must be identified." } ] }, { "incidentId": "gtg-50020", "framing": "The report describes malicious instructions reaching an AI evaluation environment and causing keys to be released.", "relevance": "direct", "connections": [ { "pillarId": "agents", "title": "Keep outside text from authorizing actions", "why": "Treat material the AI examines as untrusted content, and check sensitive tool actions against permissions enforced outside the model.", "limitation": "Prompt filtering can miss disguised instructions. Permissions must still limit the consequences when the model follows one." }, { "pillarId": "cloud", "title": "Keep keys out of reach", "why": "Give evaluation tasks limited credentials and avoid exposing production secrets to their working environment; revoke keys when their release is detected.", "limitation": "A task may legitimately need some access. Its remaining permissions still need limits and monitoring." }, { "pillarId": "software", "title": "Test the whole evaluation workflow", "why": "Review how submitted content reaches tools, files, and external connections, then test whether those boundaries hold under malicious inputs.", "limitation": "Passing a set of test inputs does not prove that every future instruction attack will be blocked." } ] }, { "incidentId": "gtg-50029", "framing": "The reported theft fed a searchable doxxing service, connecting a systems incident to the privacy and safety of real people.", "relevance": "direct", "connections": [ { "pillarId": "cloud", "title": "Reduce access to personal records", "why": "Restrict who and what can retrieve identifying information, separate sensitive collections, and review unusual bulk access or downloads.", "limitation": "Data already made public or copied elsewhere cannot be protected simply by changing permissions on the original system." }, { "pillarId": "operations", "title": "Plan for people, too", "why": "An incident response should identify whose information was exposed and coordinate technical containment with privacy, communications, and personal-safety support.", "limitation": "Technical containment may stop further access, but it cannot guarantee removal of copies or prevent every misuse of published information." } ] }, { "incidentId": "gtg-15001", "framing": "This is a reported consumer-deception case, with indirect lessons for organizational trust and awareness rather than a demonstrated enterprise break-in.", "relevance": "indirect", "connections": [ { "pillarId": "employees", "title": "Verify consequential requests", "why": "The broader awareness lesson is that a convincing conversation is not proof of identity; verify requests for money or sensitive information through a trusted channel.", "limitation": "The case did not establish a workplace payment scam. This is a transferable lesson, not the reported attack sequence." }, { "pillarId": "operations", "title": "Give people a reporting route", "why": "If suspected impersonation reaches work, staff need a clear way to report it so the organization can assess any account, payment, or information exposure.", "limitation": "Enterprise monitoring does not resolve deceptive dating services or cover personal conversations outside the organization's systems." } ] }, { "incidentId": "emerald-sleet-2024", "framing": "The providers observed AI-assisted research and drafting likely intended for deceptive emails, without establishing a successful AI-caused break-in.", "relevance": "direct", "connections": [ { "pillarId": "employees", "title": "Check identity and requests", "why": "Assess who is asking, what they want, and whether the request fits the relationship; verify sensitive requests separately even when the writing sounds natural.", "limitation": "Fluent writing is neither proof of legitimacy nor proof of AI use. Training alone cannot identify every deceptive message." }, { "pillarId": "operations", "title": "Investigate suspicious communications", "why": "Email and identity security tools can help connect a suspicious message with related messages, sign-ins, or account changes for investigation.", "limitation": "This reporting describes preparation. It does not establish which messages reached victims or which defensive tools were present." } ] }, { "incidentId": "gtg-2002-2025", "framing": "Anthropic reports intrusions, theft of private records, and ransom demands, with AI helping a human operator carry out the work.", "relevance": "direct", "connections": [ { "pillarId": "cloud", "title": "Limit access to valuable data", "why": "Map where medical and financial records are stored, restrict the accounts that can retrieve them, and watch for unusual exports.", "limitation": "Access may look legitimate when credentials are stolen. The published summary does not establish every initial entry route." }, { "pillarId": "operations", "title": "Practice the extortion response", "why": "Prepare to contain access, preserve evidence, determine what was taken, and coordinate with legal, privacy, and communications teams.", "limitation": "Backups can help restore operations but do not undo information theft or make an extortion threat disappear." } ] }, { "incidentId": "gtg-1002-2025", "framing": "In Anthropic's account, humans selected targets while groups of AI agents performed much of the intrusion work.", "relevance": "direct", "connections": [ { "pillarId": "software", "title": "Shorten the time to a verified fix", "why": "Use code analysis and testing to find exposed weaknesses, then review, deploy, and verify repairs; AI can assist with investigation and proposed changes.", "limitation": "An AI-generated repair may be wrong or incomplete. Verification must check both security and normal application behavior." }, { "pillarId": "cloud", "title": "Contain each compromised account", "why": "Narrow account permissions and separate systems so that a stolen credential provides less access to other services and sensitive records.", "limitation": "This limits possible spread; it does not establish that the specific intrusions in the report would have been prevented." }, { "pillarId": "operations", "title": "Join the evidence quickly", "why": "Connect application, identity, and data-access records so investigators can recognize related actions and act before further access occurs.", "limitation": "Targets generally cannot see the attacker's AI instructions. Detection must work from activity visible in their own systems." } ] }, { "incidentId": "cloud-credential-harvesting-2026", "framing": "Editorial defense mapping: control what runs in your cloud and what an exposed credential can reach.", "relevance": "direct", "connections": [ { "pillarId": "cloud", "title": "Notice unauthorized cloud activity", "why": "Connect workload, sign-in and network events; investigate unfamiliar running programs and unexpected outbound activity. Revoke compromised identities and isolate affected systems.", "limitation": "Traffic volume alone is not proof of an attack. Useful detection depends on coverage and the context of legitimate work." }, { "pillarId": "software", "title": "Reduce exposed entry points", "why": "Inventory internet-facing applications, investigate suspected flaws, and verify deployed repairs. Review which applications can read sensitive credentials.", "limitation": "This is general defensive relevance; the publication does not establish the initial cloud-compromise method or a specific missing patch." }, { "pillarId": "operations", "title": "Turn an alert into containment", "why": "Rehearse how responders connect suspicious events, preserve evidence, and stop unauthorized activity under a clear approval policy.", "limitation": "The victim cannot set policies for an attacker's AI. Fast triage still requires validated evidence and authority to act." } ] } ] }