Amaka Peace Onebunne

::: {#d06d201f .cell _cell_guid=‘b1076dfc-b9ad-4769-8c92-a6c4dae69d19’ _uuid=‘8f2839f25d086af736a60e9eeb907d3b93b6e0e5’ quarto-private-1=‘{“key”:“papermill”,“value”:{“duration”:1.339629,“end_time”:“2026-04-06T22:05:45.928588+00:00”,“exception”:false,“start_time”:“2026-04-06T22:05:44.588959+00:00”,“status”:“completed”}}’ tags=‘[]’ execution_count=1}

# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session

/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-Policy-FSF-TechnicalReport-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-Policy-FSF-v1-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/Google-Report-AIPrinciples-Progress-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/Google-Report-AIResponsibility-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-ModelCard-Gemini2FlashLite-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-TechReport-Imagen3-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-ModelCard-Gemini2.5Flash-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/Google-Report-AIPrinciples-Progress-2022.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/Google-Report-AIResponsibility-Feb2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-ModelCard-Gemini2Flash-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-TechReport-Gemini1-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-ModelCard-Gemini2.5Pro-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-Policy-FSF-v2-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-SafetyReport-Gemini3Pro-FSF-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-Policy-FSF-v3-Sept2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-TechReport-Gemini1.5-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-ModelCard-LLaMA2-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-ModelCard-LLaMA3.1-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-Guide-ResponsibleUse-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-Audit-CivilRights-2022.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-Assessment-HRIA-E2EE-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-ModelCard-LLaMA3.2-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-GovResponse-NTIA-RFC-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-ModelCard-LLaMA3-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-Report-ResponsibleBusinessPractices-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-o3-o4mini-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-Policy-VoluntaryCommitments-Paris-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-Operator-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT4o-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT4V-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-Policy-ExternalRedTeaming-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT4-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-o1-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-TechnicalReport-GPT-4-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT5-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT4.5-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-Policy-PreparednessFramework-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Report-RAITransparency-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Report-RAIMaturityModel-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-SystemCard-Phi3-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Standard-RAI-v2-2022.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Report-RAITransparency-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Guide-RAIImpactAssessment-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Policy-VoluntaryCommitments-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-VoluntaryCommitments-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude3.7-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-RSP-v1-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-GovResponse-OSTP-RFI-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-RSP-v2.2-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude3-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Paper-ConstitutionalAI-2022.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-ASL3-DeploymentSafeguards-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude3-Addendum-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-SystemCard-Claude4-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude2-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-RSP-v2-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-RiskReport-Pilot-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude3-OctoberAddendum-2024.pdf
/kaggle/input/notebooks/amakapeace/rai-project/__results__.html
/kaggle/input/notebooks/amakapeace/rai-project/__notebook__.ipynb
/kaggle/input/notebooks/amakapeace/rai-project/__output__.json
/kaggle/input/notebooks/amakapeace/rai-project/corpus_metadata.csv
/kaggle/input/notebooks/amakapeace/rai-project/custom.css
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_047.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_032.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_045.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_051.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_029.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_056.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_002.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_038.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_052.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_042.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_023.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_009.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_021.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_053.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_055.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_020.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_044.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_012.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_041.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_034.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_013.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_037.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_039.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_006.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_049.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_018.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_008.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_022.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_057.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_040.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_003.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_001.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_026.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_011.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_015.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_050.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_027.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_030.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_007.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_000.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_054.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_031.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_028.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_017.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_014.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_036.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_024.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_025.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_019.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_048.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_016.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_046.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_005.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_035.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_033.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_004.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_010.txt
/kaggle/input/notebooks/amakapeace/rai-project/clean_texts/doc_043.txt
/kaggle/input/notebooks/amakapeace/rai-project/__results___files/__results___26_1.png
/kaggle/input/notebooks/amakapeace/rai-project/__results___files/__results___27_0.png
/kaggle/input/notebooks/amakapeace/rai-project/__results___files/__results___26_3.png
/kaggle/input/notebooks/amakapeace/rai-project/__results___files/__results___26_2.png
/kaggle/input/notebooks/amakapeace/rai-project/__results___files/__results___26_0.png
/kaggle/input/notebooks/amakapeace/rai-project/__results___files/__results___28_0.png
/kaggle/input/notebooks/amakapeace/rai-project/.virtual_documents/__notebook_source__.ipynb

:::

# Install libraries believe I need for PDF extraction and text processing.
# Kaggle already includes many packages, so we only install what we need.

!pip install pdfplumber spacy --quiet
!pip install pytesseract pdf2image --quiet

# Download the spaCy English model used for text processing.
# Safe to run even if it is already installed.

!python -m spacy download en_core_web_sm

# Import libraries for data handling, text processing, and modeling.

import os
import re

import pandas as pd
import numpy as np
import subprocess
import sys

subprocess.run([sys.executable, "-m", "pip", "install", "pdfminer.six", "--quiet"])

import pdfplumber



import spacy

import nltk
from nltk.tokenize import sent_tokenize
nltk.download('punkt', quiet=True)

import statsmodels.api as sm
import statsmodels.formula.api as smf

import matplotlib.pyplot as plt
import seaborn as sns

from collections import Counter, defaultdict

# Load spaCy model and allow large documents

nlp = spacy.load("en_core_web_sm")
nlp.max_length = 2000000

print("Setup complete.")

     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 43.6/43.6 kB 2.0 MB/s eta 0:00:00

     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 68.1/68.1 kB 3.7 MB/s eta 0:00:00

   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.0/60.0 kB 3.5 MB/s eta 0:00:00

   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 6.6/6.6 MB 93.2 MB/s eta 0:00:00

   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.6/3.6 MB 78.5 MB/s eta 0:00:00

Collecting en-core-web-sm==3.8.0

  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl (12.8 MB)

     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.8/12.8 MB 79.7 MB/s eta 0:00:00

✔ Download and installation successful

You can now load the package via spacy.load('en_core_web_sm')

⚠ Restart to reload dependencies

If you are in a Jupyter or Colab notebook, you may need to restart Python in

order to load all the package's dependencies. You can do this by selecting the

'Restart kernel' or 'Restart runtime' option.

Setup complete.

# Set file paths for input PDFs and output files

# Base folder where my PDFs are stored in Kaggle

PDF_FOLDER = "/kaggle/input/datasets/amakapeace/raiusedpdf/"



COMPANY_FOLDERS = {
    "Anthropic":       "Anthropic",
    "OpenAI":          "OpenAI",
    "GoogleDeepMind":  "GoogleDeepMind",  
    "Microsoft":       "Microsoft",
    "Meta":            "Meta",            
}


# Folder where processed outputs will be saved
OUTPUT_FOLDER = "/kaggle/working"


# Create output directories 

os.makedirs(OUTPUT_FOLDER, exist_ok=True)
os.makedirs(os.path.join(OUTPUT_FOLDER, "clean_texts"), exist_ok=True)

# Quick check: print all PDF paths so you can verify folder names
print("PDF files found in dataset:")
for root, dirs, files in os.walk(PDF_FOLDER):
    for f in files:
        if f.endswith(".pdf"):
            print(os.path.join(root, f))

PDF files found in dataset:
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-Policy-FSF-TechnicalReport-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-Policy-FSF-v1-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/Google-Report-AIPrinciples-Progress-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/Google-Report-AIResponsibility-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-ModelCard-Gemini2FlashLite-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-TechReport-Imagen3-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-ModelCard-Gemini2.5Flash-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/Google-Report-AIPrinciples-Progress-2022.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/Google-Report-AIResponsibility-Feb2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-ModelCard-Gemini2Flash-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-TechReport-Gemini1-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-ModelCard-Gemini2.5Pro-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-Policy-FSF-v2-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-SafetyReport-Gemini3Pro-FSF-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-Policy-FSF-v3-Sept2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind/GoogleDeepMind-TechReport-Gemini1.5-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-ModelCard-LLaMA2-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-ModelCard-LLaMA3.1-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-Guide-ResponsibleUse-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-Audit-CivilRights-2022.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-Assessment-HRIA-E2EE-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-ModelCard-LLaMA3.2-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-GovResponse-NTIA-RFC-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-ModelCard-LLaMA3-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Meta/Meta-Report-ResponsibleBusinessPractices-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-o3-o4mini-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-Policy-VoluntaryCommitments-Paris-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-Operator-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT4o-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT4V-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-Policy-ExternalRedTeaming-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT4-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-o1-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-TechnicalReport-GPT-4-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT5-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-SystemCard-GPT4.5-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI/OpenAI-Policy-PreparednessFramework-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Report-RAITransparency-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Report-RAIMaturityModel-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-SystemCard-Phi3-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Standard-RAI-v2-2022.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Report-RAITransparency-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Guide-RAIImpactAssessment-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft/Microsoft-Policy-VoluntaryCommitments-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-VoluntaryCommitments-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude3.7-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-RSP-v1-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-GovResponse-OSTP-RFI-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-RSP-v2.2-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude3-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Paper-ConstitutionalAI-2022.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-ASL3-DeploymentSafeguards-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude3-Addendum-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-SystemCard-Claude4-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude2-2023.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-Policy-RSP-v2-2024.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-RiskReport-Pilot-2025.pdf
/kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic/Anthropic-ModelCard-Claude3-OctoberAddendum-2024.pdf

# Check folder structure (important for debugging)

for root, dirs, files in os.walk(PDF_FOLDER):
    print("Folder:", root)
    print("Subfolders:", dirs)
    print("Files:", len(files))
    print("-" * 40)

Folder: /kaggle/input/datasets/amakapeace/raiusedpdf/
Subfolders: ['GoogleDeepMind', 'Meta', 'OpenAI', 'Microsoft', 'Anthropic']
Files: 0
----------------------------------------
Folder: /kaggle/input/datasets/amakapeace/raiusedpdf/GoogleDeepMind
Subfolders: []
Files: 16
----------------------------------------
Folder: /kaggle/input/datasets/amakapeace/raiusedpdf/Meta
Subfolders: []
Files: 9
----------------------------------------
Folder: /kaggle/input/datasets/amakapeace/raiusedpdf/OpenAI
Subfolders: []
Files: 12
----------------------------------------
Folder: /kaggle/input/datasets/amakapeace/raiusedpdf/Microsoft
Subfolders: []
Files: 7
----------------------------------------
Folder: /kaggle/input/datasets/amakapeace/raiusedpdf/Anthropic
Subfolders: []
Files: 14
----------------------------------------

# Step 2: Build the corpus metadata table
#
# I define each document manually to ensure full control over:
# - company
# - year
# - document type
#
# Each row represents one document in the corpus.
# The filename must match exactly with the uploaded PDF file in Kaggle.
#
# The document types reflect the categories I use in my analysis:
# model_card, system_card, policy_governance,
# transparency_report, technical_report,
# safety_report, gov_response, audit_report, assessment   

DOCUMENTS = [
    # Anthropic (14 documents)
    {"filename": "Anthropic-Paper-ConstitutionalAI-2022.pdf",            "company": "Anthropic",      "year": 2022, "doc_type": "technical_report"},
    {"filename": "Anthropic-ModelCard-Claude2-2023.pdf",                 "company": "Anthropic",      "year": 2023, "doc_type": "model_card"},
    {"filename": "Anthropic-Policy-RSP-v1-2023.pdf",                     "company": "Anthropic",      "year": 2023, "doc_type": "policy_governance"},
    {"filename": "Anthropic-ModelCard-Claude3-2024.pdf",                 "company": "Anthropic",      "year": 2024, "doc_type": "model_card"},
    {"filename": "Anthropic-ModelCard-Claude3-Addendum-2024.pdf",        "company": "Anthropic",      "year": 2024, "doc_type": "model_card"},
    {"filename": "Anthropic-ModelCard-Claude3-OctoberAddendum-2024.pdf", "company": "Anthropic",      "year": 2024, "doc_type": "model_card"},
    {"filename": "Anthropic-Policy-RSP-v2-2024.pdf",                     "company": "Anthropic",      "year": 2024, "doc_type": "policy_governance"},
    {"filename": "Anthropic-Policy-VoluntaryCommitments-2024.pdf",       "company": "Anthropic",      "year": 2024, "doc_type": "policy_governance"},
    {"filename": "Anthropic-ModelCard-Claude3.7-2025.pdf",               "company": "Anthropic",      "year": 2025, "doc_type": "model_card"},
    {"filename": "Anthropic-Policy-RSP-v2.2-2025.pdf",                   "company": "Anthropic",      "year": 2025, "doc_type": "policy_governance"},
    {"filename": "Anthropic-Policy-ASL3-DeploymentSafeguards-2025.pdf",  "company": "Anthropic",      "year": 2025, "doc_type": "policy_governance"},
    {"filename": "Anthropic-SystemCard-Claude4-2025.pdf",                "company": "Anthropic",      "year": 2025, "doc_type": "system_card"},
    {"filename": "Anthropic-GovResponse-OSTP-RFI-2025.pdf",              "company": "Anthropic",      "year": 2025, "doc_type": "gov_response"},
    {"filename": "Anthropic-RiskReport-Pilot-2025.pdf",                  "company": "Anthropic",      "year": 2025, "doc_type": "safety_report"},
    
    # OpenAI (12 documents)
    {"filename": "OpenAI-SystemCard-GPT4-2023.pdf",                      "company": "OpenAI",         "year": 2023, "doc_type": "system_card"},
    {"filename": "OpenAI-SystemCard-GPT4V-2023.pdf",                     "company": "OpenAI",         "year": 2023, "doc_type": "system_card"},
    {"filename": "OpenAI-Policy-PreparednessFramework-2023.pdf",         "company": "OpenAI",         "year": 2023, "doc_type": "policy_governance"},
    {"filename": "OpenAI-TechnicalReport-GPT-4-2023.pdf",                "company": "OpenAI",         "year": 2023, "doc_type": "technical_report"},
    {"filename": "OpenAI-SystemCard-GPT4o-2024.pdf",                     "company": "OpenAI",         "year": 2024, "doc_type": "system_card"},
    {"filename": "OpenAI-SystemCard-o1-2024.pdf",                        "company": "OpenAI",         "year": 2024, "doc_type": "system_card"},
    {"filename": "OpenAI-Policy-ExternalRedTeaming-2024.pdf",            "company": "OpenAI",         "year": 2024, "doc_type": "technical_report"},
    {"filename": "OpenAI-SystemCard-Operator-2025.pdf",                  "company": "OpenAI",         "year": 2025, "doc_type": "system_card"},
    {"filename": "OpenAI-SystemCard-GPT4.5-2025.pdf",                    "company": "OpenAI",         "year": 2025, "doc_type": "system_card"},
    {"filename": "OpenAI-SystemCard-o3-o4mini-2025.pdf",                 "company": "OpenAI",         "year": 2025, "doc_type": "system_card"},
    {"filename": "OpenAI-SystemCard-GPT5-2025.pdf",                      "company": "OpenAI",         "year": 2025, "doc_type": "system_card"},
    {"filename": "OpenAI-Policy-VoluntaryCommitments-Paris-2025.pdf",    "company": "OpenAI",         "year": 2025, "doc_type": "policy_governance"},
    

    # Google DeepMind (16 documents)
    {"filename": "Google-Report-AIPrinciples-Progress-2022.pdf",         "company": "GoogleDeepMind", "year": 2022, "doc_type": "transparency_report"},
    {"filename": "Google-Report-AIPrinciples-Progress-2023.pdf",         "company": "GoogleDeepMind", "year": 2023, "doc_type": "transparency_report"},
    {"filename": "GoogleDeepMind-TechReport-Gemini1-2023.pdf",           "company": "GoogleDeepMind", "year": 2023, "doc_type": "technical_report"},
    {"filename": "GoogleDeepMind-TechReport-Gemini1.5-2024.pdf",         "company": "GoogleDeepMind", "year": 2024, "doc_type": "technical_report"},
    {"filename": "Google-Report-AIResponsibility-2024.pdf",              "company": "GoogleDeepMind", "year": 2024, "doc_type": "transparency_report"},
    {"filename": "GoogleDeepMind-Policy-FSF-v1-2024.pdf",                "company": "GoogleDeepMind", "year": 2024, "doc_type": "policy_governance"},
    {"filename": "GoogleDeepMind-Policy-FSF-v2-2025.pdf",                "company": "GoogleDeepMind", "year": 2025, "doc_type": "policy_governance"},
    {"filename": "GoogleDeepMind-Policy-FSF-TechnicalReport-2025.pdf",   "company": "GoogleDeepMind", "year": 2025, "doc_type": "technical_report"},
    {"filename": "GoogleDeepMind-Policy-FSF-v3-Sept2025.pdf",            "company": "GoogleDeepMind", "year": 2025, "doc_type": "policy_governance"},
    {"filename": "GoogleDeepMind-SafetyReport-Gemini3Pro-FSF-2025.pdf",  "company": "GoogleDeepMind", "year": 2025, "doc_type": "system_card"},
    {"filename": "Google-Report-AIResponsibility-Feb2025.pdf",           "company": "GoogleDeepMind", "year": 2025, "doc_type": "transparency_report"},
    {"filename": "GoogleDeepMind-ModelCard-Gemini2.5Pro-2025.pdf",       "company": "GoogleDeepMind", "year": 2025, "doc_type": "model_card"},
    {"filename": "GoogleDeepMind-ModelCard-Gemini2.5Flash-2025.pdf",     "company": "GoogleDeepMind", "year": 2025, "doc_type": "model_card"},
    {"filename": "GoogleDeepMind-ModelCard-Gemini2Flash-2025.pdf",       "company": "GoogleDeepMind", "year": 2025, "doc_type": "model_card"},
    {"filename": "GoogleDeepMind-ModelCard-Gemini2FlashLite-2025.pdf",   "company": "GoogleDeepMind", "year": 2025, "doc_type": "model_card"},
    {"filename": "GoogleDeepMind-TechReport-Imagen3-2024.pdf",           "company": "GoogleDeepMind", "year": 2024, "doc_type": "technical_report"},
    
    
    # Microsoft (7 documents)
    {"filename": "Microsoft-Standard-RAI-v2-2022.pdf",                   "company": "Microsoft",      "year": 2022, "doc_type": "policy_governance"},
    {"filename": "Microsoft-Report-RAIMaturityModel-2023.pdf",           "company": "Microsoft",      "year": 2023, "doc_type": "transparency_report"},
    {"filename": "Microsoft-Policy-VoluntaryCommitments-2023.pdf",       "company": "Microsoft",      "year": 2023, "doc_type": "policy_governance"},
    {"filename": "Microsoft-Guide-RAIImpactAssessment-2023.pdf",         "company": "Microsoft",      "year": 2023, "doc_type": "policy_governance"},
    {"filename": "Microsoft-SystemCard-Phi3-2024.pdf",                   "company": "Microsoft",      "year": 2024, "doc_type": "system_card"},
    {"filename": "Microsoft-Report-RAITransparency-2024.pdf",            "company": "Microsoft",      "year": 2024, "doc_type": "transparency_report"},
    {"filename": "Microsoft-Report-RAITransparency-2025.pdf",            "company": "Microsoft",      "year": 2025, "doc_type": "transparency_report"},
    
    # Meta (9 documents)
    {"filename": "Meta-Audit-CivilRights-2022.pdf",                      "company": "Meta",           "year": 2022, "doc_type": "audit_report"},
    {"filename": "Meta-ModelCard-LLaMA2-2023.pdf",                       "company": "Meta",           "year": 2023, "doc_type": "model_card"},
    {"filename": "Meta-Assessment-HRIA-E2EE-2023.pdf",                   "company": "Meta",           "year": 2023, "doc_type": "assessment"},
    {"filename": "Meta-ModelCard-LLaMA3-2024.pdf",                       "company": "Meta",           "year": 2024, "doc_type": "model_card"},
    {"filename": "Meta-ModelCard-LLaMA3.1-2024.pdf",                     "company": "Meta",           "year": 2024, "doc_type": "model_card"},
    {"filename": "Meta-ModelCard-LLaMA3.2-2024.pdf",                     "company": "Meta",           "year": 2024, "doc_type": "model_card"},
    {"filename": "Meta-Guide-ResponsibleUse-2024.pdf",                   "company": "Meta",           "year": 2024, "doc_type": "policy_governance"},
    {"filename": "Meta-GovResponse-NTIA-RFC-2024.pdf",                   "company": "Meta",           "year": 2024, "doc_type": "gov_response"},
    {"filename": "Meta-Report-ResponsibleBusinessPractices-2024.pdf",    "company": "Meta",           "year": 2024, "doc_type": "transparency_report"},
]

# Convert to dataframe

metadata_df = pd.DataFrame(DOCUMENTS)

# Basic checks to confirm everything loaded correctly

print("Total documents:", len(metadata_df))
print(metadata_df['company'].value_counts())
print(metadata_df['doc_type'].value_counts())

Total documents: 58
company
GoogleDeepMind    16
Anthropic         14
OpenAI            12
Meta               9
Microsoft          7
Name: count, dtype: int64
doc_type
policy_governance      14
model_card             13
system_card            11
transparency_report     8
technical_report        7
gov_response            2
safety_report           1
audit_report            1
assessment              1
Name: count, dtype: int64

# During initial inspection, I observed severe spacing and formatting issues
# in text extracted using standard PDF tools. These included missing word
# boundaries and broken line merges, which would compromise tokenization
# and topic modeling.
#
# To address this, I introduced an intermediate reconstruction step that
# restores spacing and removes structural artifacts before applying
# standard cleaning. This ensures that downstream analysis operates on
# linguistically meaningful text rather than raw extraction output.

import os
import re
import pandas as pd
from pdfminer.high_level import extract_text

# Optional OCR (will only run if available)
try:
    from pdf2image import convert_from_path
    import pytesseract
    OCR_AVAILABLE = True
except:
    OCR_AVAILABLE = False


# Step 3: Extract and Clean Text From PDFs

# This step reconstructs usable text from PDF documents.
# PDF extraction is inherently lossy, so this pipeline is designed not just
# to extract text, but to repair common structural issues such as:
# - missing spaces between words
# - broken line merges
# - encoding artifacts
#
# The goal is to produce text that preserves semantic units for downstream
# analysis (topic modeling, sentence parsing), rather than simply dumping raw text.



# 1. PDF Text Extraction
#
# Using pdfminer instead of pdfplumber because:
# - better reconstruction of text flow
# - more reliable spacing
# - handles multi-column layouts better
#
# This is not perfect, but it reduces the most severe extraction failures.

def extract_text_from_pdf(filepath):
    try:
        text = extract_text(filepath)
        return text if text else ""
    except Exception as e:
        print(f"  Extraction error: {e}")
        return ""

#2. OCR fallback

def extract_text_with_ocr(filepath):
    if not OCR_AVAILABLE:
        return ""
    
    try:
        images = convert_from_path(filepath)
        text = ""
        for img in images:
            text += pytesseract.image_to_string(img) + "\n"
        return text
    except Exception as e:
        print(f"  OCR failed: {e}")
        return ""
        
# 2. Structural Fixes
#
# These fixes address known PDF extraction issues:
# - words glued together (Thisistext → This is text)
# - broken hyphenation across lines
# - inconsistent spacing
#
# This step is essential before any NLP cleaning.

def fix_pdf_artifacts(text):
    if not text:
        return ""

    # Fix glued words: ThisisText → This is Text
    text = re.sub(r'([a-z])([A-Z])', r'\1 \2', text)

    # Fix hyphenation across lines
    text = re.sub(r'-\s*\n\s*', '', text)

    # Replace line breaks with space
    text = text.replace('\n', ' ')

    # Remove bullet symbols
    text = re.sub(r'[•▪■●]', ' ', text)

        
    return text.strip()


# 3. Advanced Cleaning from my (peace_genie_pro R class)
#
# This extends basic cleaning by:
# - removing structural noise (headers, URLs, etc.)
# - preserving meaningful tokens
# - standardizing punctuation and encoding
#
# Important:
def clean_text(text):
    
    # This function performs conservative cleaning of extracted PDF text.
    # The goal is to normalize and remove clear noise while preserving
    # as much semantic content as possible for downstream analysis.
    
    if not text:
        return ""

    #1. Remove Obvious noise
    # Remove URLs
    text = re.sub(r'https?://\S+|www\.\S+', ' ', text)

    # Remove Emails
    text = re.sub(r'\b[\w\.-]+@[\w\.-]+\.\w+\b', ' ', text)

    # HTML tags
    text = re.sub(r'<[^>]+>', ' ', text)

    # Standalone numbers (page numbers, figure numbers)
    text = re.sub(r'\b\d+\b', ' ', text)


    
# Remove reference section 
   # During inspection, I found that aggressive section trimming
# (e.g., removing content after "references") unintentionally
# truncated shorter policy-style documents that do not follow
# standard academic structure. Anthropic-Paper-ConstitutionalAI-2022.pdf, for instance, 
# went from 852 word count from my first documented meta data csv to 81. I have to manually cross-check.
#
# This showed that section-based trimming must be applied
# cautiously and only to longer documents with clear structural markers. So, I am adding a more cautious step:
    # Trim trailing sections only if they appear late in the document

    # 3. Conservative section trimming
    # This avoids truncating shorter or policy-style documents.
    # Only trim if:
    # 1. marker exists
    # 2. it appears in the LAST 30% of the document (likely a footer section)
    # 3. the remaining text is still reasonably long
    # Trim trailing sections only if they appear late in the document
    
    trim_markers = ["references", "bibliography", "works cited", "about us"]
    text_lower = text.lower()

    for marker in trim_markers:
        idx = text_lower.find(marker)
    
      
        if idx != -1:
            if idx > len(text) * 0.7 and idx > 300: 
            # Only trim if marker appears in last 30% of document
            # And ensure enough content remains
                text = text[:idx]
                break  

    # 4. Normalize whitespace
   
    text = re.sub(r'\s+', ' ', text).strip()

    return text


# 5. Pipeline Execution

print("Extracting and cleaning text from PDFs...\n")

rows    = []
missing = []

for doc in DOCUMENTS:
    folder   = doc['company']
    filename = doc['filename']
    company  = doc['company']
    year     = doc['year']
    doc_type = doc['doc_type']

    filepath = os.path.join(PDF_FOLDER, folder, filename)

    if not os.path.exists(filepath):
        print(f"  NOT FOUND: {filepath}")
        missing.append(filepath)
        continue

    # --- Step 1: Extract
    raw_text = extract_text_from_pdf(filepath)

    # --- Step 1b: OCR fallback (only if extraction is poor)
    if len(raw_text.split()) < 200 and OCR_AVAILABLE:
        print(f"  ⚠️ Low text detected → OCR: {filename}")
        raw_text = extract_text_with_ocr(filepath)

    # --- Step 2: Fix structural PDF issues
    fixed_text = fix_pdf_artifacts(raw_text)

    # --- Step 3: Clean and normalize
    clean = clean_text(fixed_text)

    wc = len(clean.split())
    unique_words = len(set(clean.split()))


    # Drop only clearly unusable documents

    # I do NOT remove documents simply for being short.
# Instead, I am removinh documents that are too small to carry meaningful content,
# which typically indicates extraction failure.
    
    # --- Step 4: Drop low-quality or unusable documents

    if wc < 50 or unique_words < 20:
        print(f"  ⚠️ Dropped (low content quality): {filename}")
        continue
    
    # --- Step 5: Save text
    idx      = len(rows)
    txt_path = os.path.join(OUTPUT_FOLDER, "clean_texts", f"doc_{idx:03d}.txt")

    with open(txt_path, 'w', encoding='utf-8') as f:
        f.write(clean)


    rows.append({
    "doc_id": idx,
    "filename": filename,
    "company": company,
    "year": year,
    "doc_type": doc_type,
    "word_count": wc,
    "txt_filename": f"doc_{idx:03d}.txt",
    "raw_text": clean  
})

    print(f"  OK: {filename[:60]} ({wc:,} words)")



# 6. Save Metadata

metadata_df = pd.DataFrame(rows)

print(f"\nExtracted: {len(metadata_df)} documents")
print(f"Missing:   {len(missing)} files")

metadata_df.to_csv(
    os.path.join(OUTPUT_FOLDER, "corpus_metadata.csv"),
    index=False
)

print("\nSaved corpus_metadata.csv")
print("Saved clean text files to clean_texts/")

Extracting and cleaning text from PDFs...

  OK: Anthropic-Paper-ConstitutionalAI-2022.pdf (760 words)
  OK: Anthropic-ModelCard-Claude2-2023.pdf (5,741 words)
  OK: Anthropic-Policy-RSP-v1-2023.pdf (10,439 words)
  OK: Anthropic-ModelCard-Claude3-2024.pdf (17,264 words)
  OK: Anthropic-ModelCard-Claude3-Addendum-2024.pdf (2,288 words)
  OK: Anthropic-ModelCard-Claude3-OctoberAddendum-2024.pdf (5,057 words)
  OK: Anthropic-Policy-RSP-v2-2024.pdf (9,473 words)
  OK: Anthropic-Policy-VoluntaryCommitments-2024.pdf (3,217 words)
  OK: Anthropic-ModelCard-Claude3.7-2025.pdf (13,046 words)
  OK: Anthropic-Policy-RSP-v2.2-2025.pdf (9,975 words)
  OK: Anthropic-Policy-ASL3-DeploymentSafeguards-2025.pdf (7,321 words)

Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats

  OK: Anthropic-SystemCard-Claude4-2025.pdf (32,777 words)
  OK: Anthropic-GovResponse-OSTP-RFI-2025.pdf (3,639 words)
  OK: Anthropic-RiskReport-Pilot-2025.pdf (21,985 words)
  OK: OpenAI-SystemCard-GPT4-2023.pdf (27,649 words)
  OK: OpenAI-SystemCard-GPT4V-2023.pdf (5,061 words)
  OK: OpenAI-Policy-PreparednessFramework-2023.pdf (5,957 words)
  OK: OpenAI-TechnicalReport-GPT-4-2023.pdf (22,926 words)
  OK: OpenAI-SystemCard-GPT4o-2024.pdf (11,785 words)
  OK: OpenAI-SystemCard-o1-2024.pdf (12,422 words)
  OK: OpenAI-Policy-ExternalRedTeaming-2024.pdf (7,040 words)
  OK: OpenAI-SystemCard-Operator-2025.pdf (5,086 words)
  OK: OpenAI-SystemCard-GPT4.5-2025.pdf (8,492 words)
  OK: OpenAI-SystemCard-o3-o4mini-2025.pdf (9,516 words)
  OK: OpenAI-SystemCard-GPT5-2025.pdf (19,195 words)
  OK: OpenAI-Policy-VoluntaryCommitments-Paris-2025.pdf (1,781 words)
  OK: Google-Report-AIPrinciples-Progress-2022.pdf (12,436 words)
  OK: Google-Report-AIPrinciples-Progress-2023.pdf (12,499 words)
  OK: GoogleDeepMind-TechReport-Gemini1-2023.pdf (33,073 words)
  OK: GoogleDeepMind-TechReport-Gemini1.5-2024.pdf (53,349 words)

Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats

  OK: Google-Report-AIResponsibility-2024.pdf (4,774 words)

Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats
Could not get FontBBox from font descriptor because None cannot be parsed as 4 floats

  OK: GoogleDeepMind-Policy-FSF-v1-2024.pdf (1,253 words)
  OK: GoogleDeepMind-Policy-FSF-v2-2025.pdf (4,023 words)
  OK: GoogleDeepMind-Policy-FSF-TechnicalReport-2025.pdf (2,835 words)
  OK: GoogleDeepMind-Policy-FSF-v3-Sept2025.pdf (5,980 words)
  OK: GoogleDeepMind-SafetyReport-Gemini3Pro-FSF-2025.pdf (9,008 words)
  OK: Google-Report-AIResponsibility-Feb2025.pdf (6,277 words)
  OK: GoogleDeepMind-ModelCard-Gemini2.5Pro-2025.pdf (7,330 words)
  OK: GoogleDeepMind-ModelCard-Gemini2.5Flash-2025.pdf (2,967 words)
  OK: GoogleDeepMind-ModelCard-Gemini2Flash-2025.pdf (1,572 words)
  OK: GoogleDeepMind-ModelCard-Gemini2FlashLite-2025.pdf (1,556 words)
  OK: GoogleDeepMind-TechReport-Imagen3-2024.pdf (5,675 words)
  OK: Microsoft-Standard-RAI-v2-2022.pdf (6,438 words)
  OK: Microsoft-Report-RAIMaturityModel-2023.pdf (485 words)
  OK: Microsoft-Policy-VoluntaryCommitments-2023.pdf (5,412 words)
  OK: Microsoft-Guide-RAIImpactAssessment-2023.pdf (9,990 words)
  OK: Microsoft-SystemCard-Phi3-2024.pdf (5,094 words)
  OK: Microsoft-Report-RAITransparency-2024.pdf (15,111 words)
  OK: Microsoft-Report-RAITransparency-2025.pdf (16,854 words)
  OK: Meta-Audit-CivilRights-2022.pdf (39,451 words)
  OK: Meta-ModelCard-LLaMA2-2023.pdf (1,066 words)
  OK: Meta-Assessment-HRIA-E2EE-2023.pdf (7,488 words)
  OK: Meta-ModelCard-LLaMA3-2024.pdf (2,653 words)
  OK: Meta-ModelCard-LLaMA3.1-2024.pdf (2,755 words)
  OK: Meta-ModelCard-LLaMA3.2-2024.pdf (3,407 words)
  OK: Meta-Guide-ResponsibleUse-2024.pdf (6,838 words)
  OK: Meta-GovResponse-NTIA-RFC-2024.pdf (15,831 words)
  OK: Meta-Report-ResponsibleBusinessPractices-2024.pdf (13,179 words)

Extracted: 58 documents
Missing:   0 files

Saved corpus_metadata.csv
Saved clean text files to clean_texts/

# Cleaning strategy note:
#
# Earlier versions of the pipeline removed short text segments aggressively,
# which led to substantial loss of meaningful content, especially in model
# cards and policy documents where important information is often presented
# in short or structured fragments.
#
# The current approach prioritizes normalization over deletion. Rather than
# filtering out short segments, it preserves document structure while removing
# only clearly non-informative elements such as URLs, HTML artifacts, and
# reference sections.
#
# This ensures that the resulting text remains faithful to the original
# document content while still being suitable for downstream analysis.

# Extraction note:
#
# PDF extraction is inherently imperfect. Some documents lack proper text
# layers or contain complex formatting that standard parsers cannot fully
# reconstruct. To mitigate this, I combine:
# - pdfminer for primary extraction
# - structural repair (spacing and hyphenation fixes)
# - OCR fallback for severely degraded documents
#
# Documents that still fail to produce sufficient text are excluded to
# maintain data quality.

# Peek at a few documents to compare quality

for i in range(3):
    path = os.path.join(OUTPUT_FOLDER, "clean_texts", f"doc_{i:03d}.txt")
    
    print(f"\n--- doc_{i:03d} ---\n")
    
    if os.path.exists(path):
        with open(path, "r", encoding="utf-8") as f:
            print(f.read()[:500])
    else:
        print("File not found")


--- doc_000 ---

Constitutional AI: Harmlessness from AI Feedback Anthropic has uncovered a new approach to AI safety that shapes the outputs of AI systems according to a set of principles. The approach is called Constitutional AI (CAI) because it gives an AI system a set of principles (i.e., a “constitution”) against which it can evaluate its own outputs. CAI enables AI systems to generate useful responses while also minimizing harm. This is important because existing techniques for training models to mirror hu

--- doc_001 ---

Model Card and Evaluations for Claude Models Anthropic Introduction This report includes the model card [ ] for Claude models, focusing on Claude , along with the results of a range of safety, alignment, and capabilities evaluations. We have been iterating on the training and evaluation of Claude-type models since our first work on Reinforcement Learning from Human Feedback (RLHF) [ ]; the newest Claude model represents a continuous evolution from those early and less capable ‘helpful and harmle

--- doc_002 ---

Anthropic's Responsible Scaling Policy Version . Effective September , As AI models become more capable, Anthropic believes that they will create major economic and social value, but will also present increasingly severe risks. With this document we are making a public commitment to a concrete framework for managing these risks—one that will evolve over time, but that seeks to establish clear expectations and accountability in its initial form. We focus these commitments specifically on catastro

# Step 4: Since I will be using 2 notebooks (R and Python), save these clean text files or the R notebook
# The R notebook reads these .txt files to run the STM.
# Note to self: Also save the metadata CSV so R knows which file belongs to which company, year, and document type.

text_folder = os.path.join(OUTPUT_FOLDER, "clean_texts")
os.makedirs(text_folder, exist_ok=True)

for idx, row in metadata_df.iterrows():
    txt_path = os.path.join(text_folder, f"doc_{idx:03d}.txt")
    with open(txt_path, 'w', encoding='utf-8') as f:
        f.write(row['raw_text'])

metadata_df['txt_filename'] = [f"doc_{i:03d}.txt" for i in range(len(metadata_df))]

# Save metadata without the raw text column (too large for CSV)
metadata_df.drop(columns=['raw_text']).to_csv(
    os.path.join(OUTPUT_FOLDER, "corpus_metadata.csv"), index=False
)

print(f"Saved {len(metadata_df)} text files to clean_texts/")
print("Saved corpus_metadata.csv")
print("These are ready for the R notebook.")

Saved 58 text files to clean_texts/
Saved corpus_metadata.csv
These are ready for the R notebook.


# Step 5: Define Dictionary for analysis
#
# These dictionaries operationalize key concepts in the study.
# Domain dictionaries identify what a sentence is about.
# The obligation dictionary captures how strongly a commitment is expressed.
#
# The domain categories are derived from prior AI ethics and governance
# literature, which consistently identifies safety, fairness, transparency,
# privacy, and accountability as core dimensions of responsible AI  (Floridi et al., 2018; Jobin et al., 2019; Fjeld et al., 2020; Hagendorff, 2020).
# Additional governance and regulatory categories reflect the shift from ethical principles toward implementation and oversight (Bietti, 2020;
# Raji et al., 2020; Suzor, 2019; Gorwa et al., 2020).
#
# Keywords were initially selected from these conceptual categories and iteratively refined through inspection of the corpus to reflect actual
# language used in corporate Responsible AI documents.

DOMAIN_DICT = {
    "safety_risk": [
        "safety", "safe", "risk", "risks", "harm", "harms",
        "danger", "hazard", "catastrophic", "misuse", "abuse",
        "vulnerability", "threat", "unsafe", "mitigation",
        "mitigate", "evaluation", "evaluate", "testing", "test",
        "deployment", "deploy", "safeguard", "protection",
        "red team", "red-team", "preparedness", "biosecurity",
        "cbrn", "dual use", "dual-use", "frontier"
    ],

# Fairness and bias are treated as a distinct domain because prior work shows discrimination, bias, and equity concerns are central and recurring
# issues in AI systems and governance (Jobin et al., 2019; Hagendorff, 2020; Papagiannidis et al., 2023; Saheb and Izak, 2023; Correa et al., 2023;
# Kim, 2023; Tidjon et al., 2022; Seppälä et al., 2023).
    "fairness_bias": [
        "fair", "fairness", "bias", "biased", "discrimination",
        "discriminatory", "equity", "equitable", "equality",
        "stereotype", "stereotyping", "representation",
        "diversity", "inclusion", "inclusive", "marginalized",
        "underrepresented", "demographic", "protected",
        "race", "gender", "age", "disability", "intersectional"
    ],

    "transparency_disclosure": [
        "transparent", "transparency", "disclosure", "disclose",
        "explain", "explainability", "interpretability",
        "interpretable", "report", "reporting", "document",
        "documentation", "model card", "system card",
        "publish", "published", "communicate", "communication"
    ],

    "privacy_data_governance": [
        "privacy", "private", "personal data", "personal information",
        "data protection", "consent", "confidential",
        "confidentiality", "surveillance", "data collection",
        "training data", "data governance", "anonymization",
        "anonymize"
    ],

       "governance_accountability": [
        "accountability", "accountable", "responsibility",
        "responsible", "governance", "oversight", "audit",
        "auditing", "compliance", "compliant", "review",
        "monitoring", "internal controls"
    ],

    "regulatory_compliance": [
        "regulation", "regulatory", "law", "legal", "legislation",
        "policy", "standard", "framework", "comply", "compliance", "requirement",
        "nist", "eu ai act", "executive order", "government",
        "white house", "ntia", "ftc"
    ]
}

# Obligation levels are grounded in deontic modality, where modal verbs
# distinguish binding requirements from recommendations and discretionary
# language (Kiyavitskaya et al., 2008; Salathé et al., 2012).
# This distinction is important in AI governance because prior work shows
# that many corporate commitments remain aspirational rather than enforceable
# (Mittelstadt, 2019; Bietti, 2020; Raji et al., 2020).

OBLIGATION_DICT = {
    "strong": [
        "must", "shall", "required", "require", "requires",
        "mandated", "mandate", "obligated",
        "prohibited", "prohibit", "forbidden"
    ],

   
    # Moderate obligation: recommendations, expectations
    "moderate": [
        "should", "ought to", "expected to", "need to" "expected",
        "encouraged", "encourage", "recommended", "recommend",
        "advised", "advise", "important to"
    ],
    # Weak obligation: aspirational language
    "weak": [
        "may", "might", "aim to", "aim", "strive", "seek to",
        "seek", "where possible", "where appropriate",
        "endeavor", "hope to", "intend to", "intend", "try to",
        "consider", "explore"
    ]
}

print("Dictionaries defined.")
print("Domains:", list(DOMAIN_DICT.keys()))

Dictionaries defined.
Domains: ['safety_risk', 'fairness_bias', 'transparency_disclosure', 'privacy_data_governance', 'governance_accountability', 'regulatory_compliance']

# Actor dictionary retained for potential responsibility allocation analysis.
# It is not used in the main models in this paper, which focus on obligation
# strength, domain association, and positional distribution.
# This can support a follow-up analysis on how responsibility is assigned
# across organizations, developers, users, and AI systems.

ACTOR_DICT = {
        "organization": [
        "we", "our", "the company", "anthropic", "openai",
        "google", "microsoft", "meta", "deepmind",
        "the organization", "the provider", "the platform", "the lab", "the team"
    ],
    "developer": [
        "developer", "developers", "engineer", "engineers",
        "researcher", "researchers", "practitioner", "practitioners",
        "builder", "builders", "creator", "creators", "model developer"
    ],
    "user": [
        "user", "users", "end user", "end users", "customer",
        "customers", "individual", "individuals", "person",
        "people", "operator", "operators", "deployer", "deployers"
    ],
    "ai_system": [
        "the model", "the system", "the ai", "claude", "gpt",
        "llama", "gemini", "phi", "the algorithm", "the tool",
        "the technology", "the assistant", "the agent"
    ]
}

# Analyze each document at the sentence level.
# Each sentence is coded for:
#   - obligation strength (strong, moderate, weak, none)
#   - domain presence (risk, fairness, etc.)
#   - position in the document (for RQ1)

def keyword_match(text, keyword):
    """
    Match full words or phrases using regex word boundaries.
    This avoids false matches (e.g., 'risk' inside 'brisk').
    """
    pattern = r'\b' + re.escape(keyword) + r'\b'
    return re.search(pattern, text) is not None


def analyze_document(text, doc_id):
    if not text or len(text.strip()) < 10:
        return []

    # Split into sentences using spaCy
    doc = nlp(text)
    sentences = list(doc.sents)

    total_sentences = len(sentences)
    records = []

    for i, sent in enumerate(sentences):
        sent_text  = sent.text.strip()
        sent_lower = sent_text.lower()

        # Skip very short sentences (noise reduction)
        if len(sent_lower.split()) < 5:
            continue

        record = {
            "doc_id": doc_id,
            "sentence_id": i,
            "sentence": sent_text,
            "position_ratio": i / max(total_sentences - 1, 1)  # 0 to 1 scale
        }

        
        # Obligation classification
        
        obligation = "none"

        for term in OBLIGATION_DICT["strong"]:
            if keyword_match(sent_lower, term):
                obligation = "strong"
                break

        if obligation == "none":
            for term in OBLIGATION_DICT["moderate"]:
                if keyword_match(sent_lower, term):
                    obligation = "moderate"
                    break

        if obligation == "none":
            for term in OBLIGATION_DICT["weak"]:
                if keyword_match(sent_lower, term):
                    obligation = "weak"
                    break

        record["obligation_level"] = obligation

       
        # Domain detection

        for domain, keywords in DOMAIN_DICT.items():
            record[f"domain_{domain}"] = int(
                any(keyword_match(sent_lower, kw) for kw in keywords)
            )

        
        # Actor detection (will probably use later)
        
        for actor, keywords in ACTOR_DICT.items():
            record[f"actor_{actor}"] = int(
                any(keyword_match(sent_lower, kw) for kw in keywords)
            )

        records.append(record)

    return records

print("Starting sentence-level analysis. This takes several minutes.")

all_records = []

for idx, row in metadata_df.iterrows():
    print(f"Processing {idx+1}/{len(metadata_df)}: {row['filename'][:50]}")

    records = analyze_document(row['raw_text'], idx)
    all_records.extend(records)

sentences_df = pd.DataFrame(all_records)

Starting sentence-level analysis. This takes several minutes.
Processing 1/58: Anthropic-Paper-ConstitutionalAI-2022.pdf
Processing 2/58: Anthropic-ModelCard-Claude2-2023.pdf
Processing 3/58: Anthropic-Policy-RSP-v1-2023.pdf
Processing 4/58: Anthropic-ModelCard-Claude3-2024.pdf
Processing 5/58: Anthropic-ModelCard-Claude3-Addendum-2024.pdf
Processing 6/58: Anthropic-ModelCard-Claude3-OctoberAddendum-2024.p
Processing 7/58: Anthropic-Policy-RSP-v2-2024.pdf
Processing 8/58: Anthropic-Policy-VoluntaryCommitments-2024.pdf
Processing 9/58: Anthropic-ModelCard-Claude3.7-2025.pdf
Processing 10/58: Anthropic-Policy-RSP-v2.2-2025.pdf
Processing 11/58: Anthropic-Policy-ASL3-DeploymentSafeguards-2025.pd
Processing 12/58: Anthropic-SystemCard-Claude4-2025.pdf
Processing 13/58: Anthropic-GovResponse-OSTP-RFI-2025.pdf
Processing 14/58: Anthropic-RiskReport-Pilot-2025.pdf
Processing 15/58: OpenAI-SystemCard-GPT4-2023.pdf
Processing 16/58: OpenAI-SystemCard-GPT4V-2023.pdf
Processing 17/58: OpenAI-Policy-PreparednessFramework-2023.pdf
Processing 18/58: OpenAI-TechnicalReport-GPT-4-2023.pdf
Processing 19/58: OpenAI-SystemCard-GPT4o-2024.pdf
Processing 20/58: OpenAI-SystemCard-o1-2024.pdf
Processing 21/58: OpenAI-Policy-ExternalRedTeaming-2024.pdf
Processing 22/58: OpenAI-SystemCard-Operator-2025.pdf
Processing 23/58: OpenAI-SystemCard-GPT4.5-2025.pdf
Processing 24/58: OpenAI-SystemCard-o3-o4mini-2025.pdf
Processing 25/58: OpenAI-SystemCard-GPT5-2025.pdf
Processing 26/58: OpenAI-Policy-VoluntaryCommitments-Paris-2025.pdf
Processing 27/58: Google-Report-AIPrinciples-Progress-2022.pdf
Processing 28/58: Google-Report-AIPrinciples-Progress-2023.pdf
Processing 29/58: GoogleDeepMind-TechReport-Gemini1-2023.pdf
Processing 30/58: GoogleDeepMind-TechReport-Gemini1.5-2024.pdf
Processing 31/58: Google-Report-AIResponsibility-2024.pdf
Processing 32/58: GoogleDeepMind-Policy-FSF-v1-2024.pdf
Processing 33/58: GoogleDeepMind-Policy-FSF-v2-2025.pdf
Processing 34/58: GoogleDeepMind-Policy-FSF-TechnicalReport-2025.pdf
Processing 35/58: GoogleDeepMind-Policy-FSF-v3-Sept2025.pdf
Processing 36/58: GoogleDeepMind-SafetyReport-Gemini3Pro-FSF-2025.pd
Processing 37/58: Google-Report-AIResponsibility-Feb2025.pdf
Processing 38/58: GoogleDeepMind-ModelCard-Gemini2.5Pro-2025.pdf
Processing 39/58: GoogleDeepMind-ModelCard-Gemini2.5Flash-2025.pdf
Processing 40/58: GoogleDeepMind-ModelCard-Gemini2Flash-2025.pdf
Processing 41/58: GoogleDeepMind-ModelCard-Gemini2FlashLite-2025.pdf
Processing 42/58: GoogleDeepMind-TechReport-Imagen3-2024.pdf
Processing 43/58: Microsoft-Standard-RAI-v2-2022.pdf
Processing 44/58: Microsoft-Report-RAIMaturityModel-2023.pdf
Processing 45/58: Microsoft-Policy-VoluntaryCommitments-2023.pdf
Processing 46/58: Microsoft-Guide-RAIImpactAssessment-2023.pdf
Processing 47/58: Microsoft-SystemCard-Phi3-2024.pdf
Processing 48/58: Microsoft-Report-RAITransparency-2024.pdf
Processing 49/58: Microsoft-Report-RAITransparency-2025.pdf
Processing 50/58: Meta-Audit-CivilRights-2022.pdf
Processing 51/58: Meta-ModelCard-LLaMA2-2023.pdf
Processing 52/58: Meta-Assessment-HRIA-E2EE-2023.pdf
Processing 53/58: Meta-ModelCard-LLaMA3-2024.pdf
Processing 54/58: Meta-ModelCard-LLaMA3.1-2024.pdf
Processing 55/58: Meta-ModelCard-LLaMA3.2-2024.pdf
Processing 56/58: Meta-Guide-ResponsibleUse-2024.pdf
Processing 57/58: Meta-GovResponse-NTIA-RFC-2024.pdf
Processing 58/58: Meta-Report-ResponsibleBusinessPractices-2024.pdf

# This step ensures that sentence-level data is correctly aligned with
# document-level metadata.
#
# Earlier transformations may introduce duplicate or outdated metadata columns
# in the sentence-level dataframe. To avoid inconsistencies, I first drop any
# existing metadata fields and then re-merge them from the original
# metadata table using doc_id. This is also seamless, should I intend to rerun the codes
#
# This guarantees that:
# - each sentence retains the correct document-level context (company, year, etc.)
# - metadata is consistent across the pipeline
# - downstream analyses (e.g., regression, grouping) rely on a single source of truth

sentences_df = sentences_df.drop(
    columns=['company','year','doc_type','filename','word_count'],
    errors='ignore'
)

sentences_df = sentences_df.merge(
    metadata_df[['company','year','doc_type','filename','word_count']]
    .reset_index().rename(columns={'index':'doc_id'}),
    on='doc_id',
    how='left'
)

sentences_df.to_csv("sentences_analyzed.csv", index=False)

print("Total sentences:", len(sentences_df))
print(sentences_df['obligation_level'].value_counts())

Total sentences: 26254
obligation_level
none        23287
weak         1304
strong        876
moderate      787
Name: count, dtype: int64

print(sentences_df.columns.tolist())

['doc_id', 'sentence_id', 'sentence', 'position_ratio', 'obligation_level', 'domain_safety_risk', 'domain_fairness_bias', 'domain_transparency_disclosure', 'domain_privacy_data_governance', 'domain_governance_accountability', 'domain_regulatory_compliance', 'actor_organization', 'actor_developer', 'actor_user', 'actor_ai_system', 'company', 'year', 'doc_type', 'filename', 'word_count']

# Step 6: Create Docuemnt level statistic from sentence level data

# At this point, sentences_df has one row per sentence.
# Each sentence already has:
#   - an obligation label (strong, moderate, weak, or none)
#   - domain indicators (for example, domain_safety_risk = 1 or 0)
#   - actor indicators (for example, actor_organization = 1 or 0), which I mentioned I will probably use later
#
# This step collapses that sentence-level information into one row per document.
#
# Why do I need this?
# Because regression models work more naturally at the document level.
# For example, I may want one row per document with variables like:
#   - proportion of strong obligation sentences
#   - proportion of safety/risk sentences
#   - proportion of fairness/bias sentences
#   - proportion of sentences mentioning the organization
#
# These document-level measures will later help me answer:
#   - RQ4: Do domains predict obligation strength?
#   - RQ5: Do these relationships vary by company or document type?


def compute_doc_stats(sentences_df, metadata_df):
    """
    This function takes:
      1. sentences_df  -> one row per sentence
      2. metadata_df   -> one row per document with document information

    It returns:
      doc_df -> one row per document with summary statistics
    """

    # I will now store one dictionary per document in this list.
    # Later, I will turn the list into a DataFrame.
    doc_stats = []

    # Loop through each unique document id found in the sentence-level data.
    for doc_id in sentences_df['doc_id'].unique():

        # Select only the sentences that belong to the current document.
        sents = sentences_df[sentences_df['doc_id'] == doc_id]

        # Count how many sentences are in this document.
        n = len(sents)

        # Skip documents with zero sentences just in case.
        if n == 0:
            continue

        # Start a dictionary that will hold all summary values for this document.
        # This dictionary will become one row in the final document-level DataFrame.
        record = {
            "doc_id": doc_id,
            "n_sentences": n
        }

        #Control for document size in regression
        record['doc_length_log'] = np.log(n + 1)

        
        
        # Part 1: Obligation Proportions
        #
        # The column 'obligation_level' contains one of:
        #   strong, moderate, weak, none
        #
        # Here, I want to know:
        #   What proportion of this document's sentences are strong?
        #   What proportion are moderate?
        #   What proportion are weak?
        #   What proportion have no obligation language at all?

        # Count how many times each obligation level appears in this document.
        ob_counts = sents['obligation_level'].value_counts()

        # For each possible obligation level, compute the proportion.
        # Example:
        # if 20 out of 100 sentences are strong, then prop_strong_obligation = 0.20
        for level in ['strong', 'moderate', 'weak', 'none']:
            record[f"prop_{level}_obligation"] = ob_counts.get(level, 0) / n

       
        
        # Part 2: Obligation Strength Score        
        #
        # This is one of my key variables.
        #
        # Also, I do NOT want to know only how often obligation appears.
        # I want to know how strong the obligation is when it does appear.
        #
        # So I:
        #   1. keep only obligation-bearing sentences
        #      (strong, moderate, or weak)
        #   2. ask what fraction of those are strong
        #
        # Formula:
        #   obligation_strength_score =
        #       number of strong obligation sentences
        #       divided by
        #       number of all obligation-bearing sentences

        # Keep only strong, moderate, and weak sentences.
        ob_sents = sents[sents['obligation_level'].isin(['strong', 'moderate', 'weak'])]

        # Count how many obligation-bearing sentences there are.
        n_ob = len(ob_sents)

        if n_ob > 0:
            # Count how many of those obligation-bearing sentences are strong.
            n_strong = (ob_sents['obligation_level'] == 'strong').sum()

            # Compute the obligation strength score.
            record['obligation_strength_score'] = n_strong / n_ob
        else:
            # If the document has no obligation-bearing sentences,
            # I then set the score to 0.0 for now.
            record['obligation_strength_score'] = 0.0

        # Also keep the raw number of obligation-bearing sentences.
        # This is probably useful for checking whether some documents hardly contain any.
        record['n_obligation_sentences'] = n_ob

        
        # Part 3: Domain Proportions
        
        # Sentence-level domain columns look like:
        #   domain_safety_risk
        #   domain_fairness_bias
        #   domain_transparency_disclosure
        #   ...
        #
        # Each sentence has 1 if the domain appears and 0 if it does not.
        #
        # At the document level, I want:
        #   What proportion of sentences in this document mention each domain?
        #
        # Example:
        # If 30 out of 100 sentences mention safety/risk,
        # then prop_domain_safety_risk = 0.30

        # Find all columns in the sentence-level data that start with 'domain_'.
        domain_cols = [col for col in sents.columns if col.startswith('domain_')]

        # Compute a proportion for each domain column.
        for col in domain_cols:
            record[f"prop_{col}"] = sents[col].sum() / n

       
        # Part 4: Actor Proportions
        
        # These columns are optional for my main paper, but I want to keep them
        # because I may need them for another paper or just exploratory analysis.
        #
        # Sentence-level actor columns look like:
        #   actor_organization
        #   actor_developer
        #   actor_user
        #   actor_ai_system
        #
        # Again, each sentence has 1 or 0.
        # Here I am calculating the share of sentences in the document that mention each actor.
        # I do not need them for this paper, so they are commented out for now.

        #actor_cols = [col for col in sents.columns if col.startswith('actor_')]

        #for col in actor_cols:
            #record[f"prop_{col}"] = sents[col].sum() / n

        # After finishing all calculations for this document,
        # add the dictionary to the list.
        doc_stats.append(record)

    # Turn the list of dictionaries into a DataFrame.
    doc_df = pd.DataFrame(doc_stats)

    
    
    # Part 5: Merge back document Metadata
    
    # Right now, doc_df has the calculated statistics, but I also need document information such as:
    #   company
    #   year
    #   doc_type
    #   filename
    #   word_count
    #
    # I am merging those in using doc_id.

    doc_df = doc_df.merge(
        metadata_df[['company', 'year', 'doc_type', 'filename', 'word_count']]
        .reset_index()
        .rename(columns={'index': 'doc_id'}),
        on='doc_id',
        how='left'
    )

    # Return the final document-level DataFrame.
    return doc_df


# Run function and save the result in a new DataFrame.
doc_level_df = compute_doc_stats(sentences_df, metadata_df)

# Print a quick summary so I can inspect whether the output looks right.
print(f"Document-level stats created for {len(doc_level_df)} documents.")

print("\nHere are a few important columns to check:")

print(
    doc_level_df[
        [
            'filename',
            'company',
            'year',
            'doc_type',
            'obligation_strength_score',
            'prop_domain_safety_risk'
        ]
    ].head(8)
)

# Save the document-level output so I dont lose this step.
doc_level_df.to_csv(
    os.path.join(OUTPUT_FOLDER, "document_level_stats.csv"),
    index=False
)

print("\nSaved document_level_stats.csv")

Document-level stats created for 58 documents.

Here are a few important columns to check:
                                            filename    company  year  \
0          Anthropic-Paper-ConstitutionalAI-2022.pdf  Anthropic  2022   
1               Anthropic-ModelCard-Claude2-2023.pdf  Anthropic  2023   
2                   Anthropic-Policy-RSP-v1-2023.pdf  Anthropic  2023   
3               Anthropic-ModelCard-Claude3-2024.pdf  Anthropic  2024   
4      Anthropic-ModelCard-Claude3-Addendum-2024.pdf  Anthropic  2024   
5  Anthropic-ModelCard-Claude3-OctoberAddendum-20...  Anthropic  2024   
6                   Anthropic-Policy-RSP-v2-2024.pdf  Anthropic  2024   
7     Anthropic-Policy-VoluntaryCommitments-2024.pdf  Anthropic  2024   

            doc_type  obligation_strength_score  prop_domain_safety_risk  
0   technical_report                   0.000000                 0.225000  
1         model_card                   0.076923                 0.198347  
2  policy_governance                   0.290541                 0.435556  
3         model_card                   0.346939                 0.231156  
4         model_card                   0.500000                 0.307692  
5         model_card                   0.375000                 0.248299  
6  policy_governance                   0.545455                 0.478036  
7  policy_governance                   0.285714                 0.492063  

Saved document_level_stats.csv

# Keep only ONE clean version of metadata columns

sentences_df = sentences_df[
    [
        'doc_id',
        'sentence_id',
        'sentence',
        'position_ratio',
        'obligation_level',
        'domain_safety_risk',
        'domain_fairness_bias',
        'domain_transparency_disclosure',
        'domain_privacy_data_governance',
        'domain_governance_accountability',
        'domain_regulatory_compliance',
        'company',
        'year',
        'doc_type',
        'filename',
        'word_count'
    ]
]

print("Cleaned sentence-level dataframe:")
print(sentences_df.columns.tolist())

Cleaned sentence-level dataframe:
['doc_id', 'sentence_id', 'sentence', 'position_ratio', 'obligation_level', 'domain_safety_risk', 'domain_fairness_bias', 'domain_transparency_disclosure', 'domain_privacy_data_governance', 'domain_governance_accountability', 'domain_regulatory_compliance', 'company', 'year', 'doc_type', 'filename', 'word_count']

# Step 7-Exploratory Analysis: Responsibility Allocation (I will not be using this here)
#
# This section examines how responsibility is attributed in RAI documents.
# It is not part of the main research questions but provides additional
# insight into whether responsibility is assigned to organizations,
# developers, users, or the AI system itself.
#
# Actor proportions were computed earlier as the share of sentences
# mentioning each actor category within a document.

print("Exploratory Analysis: Responsibility Allocation")


# 1. Define actor variables


actor_cols = [
    'prop_actor_organization',
    'prop_actor_developer',
    'prop_actor_user',
    'prop_actor_ai_system'
]

# Automatically find actor columns if they exist
actor_cols = [col for col in doc_level_df.columns if col.startswith('prop_actor_')]

# If no actor columns exist, skip safely
if len(actor_cols) == 0:
    print("\nNo actor variables found. Skipping responsibility analysis.")
else:


# 2. Descriptive statistics

# These show how often each actor is referenced on average.
   
    
    print("\nAverage actor mention rates across full corpus:")
    print(doc_level_df[actor_cols].mean().round(3))
    
# Compare across companies
    print("\nActor mentions by company:")
    print(doc_level_df.groupby('company')[actor_cols].mean().round(3))

# Compare across document types
    print("\nActor mentions by document type:")
    print(doc_level_df.groupby('doc_type')[actor_cols].mean().round(3))


# 3. Prepare data for regression

# Convert categorical variables into dummy variables.
# This allows regression to estimate differences between companies
# and document types.

reg_df = pd.get_dummies(
    doc_level_df,
    columns=['company', 'doc_type'],
    drop_first=True
)

# Standardize word count so it is comparable to other predictors.
# This prevents large scale differences from affecting coefficients.

reg_df['word_count_std'] = (
    (reg_df['word_count'] - reg_df['word_count'].mean())
    / reg_df['word_count'].std()
)

# Identify predictor variables
company_dummies = [c for c in reg_df.columns if c.startswith('company_')]
doctype_dummies = [c for c in reg_df.columns if c.startswith('doc_type_')]

predictors = company_dummies + doctype_dummies + ['year', 'word_count_std']


# 4. Run regression models
# Separate model for each actor category.
# Dependent variable = proportion of sentences mentioning that actor.

print("\nRegression Results: What predicts actor mentions?")

rq2_results = {}

for actor_col in actor_cols:

    actor_name = actor_col.replace('prop_actor_', '').upper()
    print(f"\nActor: {actor_name}")

    # Dependent variable
    y = reg_df[actor_col]

    # Independent variables
    X = reg_df[predictors].astype(float)
    X = sm.add_constant(X)

    # Drop missing values
    mask = y.notna()

    # Run OLS with robust standard errors (HC3)
    model = sm.OLS(y[mask], X[mask]).fit(cov_type='HC3')

    rq2_results[actor_name] = model

    # Print model fit
    print(f"  R-squared: {model.rsquared:.3f} | N: {mask.sum()}")

    # Print statistically significant predictors
    sig = model.pvalues[(model.pvalues < 0.05) & (model.pvalues.index != 'const')]

    if len(sig) > 0:
        for pred in sig.index:
            print(f"  {pred}: coef={model.params[pred]:.4f}, p={model.pvalues[pred]:.3f}")
    else:
        print("  No significant predictors at p < 0.05")

Exploratory Analysis: Responsibility Allocation

No actor variables found. Skipping responsibility analysis.

Regression Results: What predicts actor mentions?

# Step 8: Domain Olbigation Relationships

# This examines:
# - how obligation strength varies across companies and document types
# - which domains are associated with strong, moderate, or weak obligation

print("DOMAIN–OBLIGATION ANALYSIS")


# Obligation strength by company


print("\nObligation strength by company:")
print(
    doc_level_df.groupby('company')['obligation_strength_score']
    .agg(['mean','std','min','max'])
    .round(3)
)


# Obligation strength by document type


print("\nObligation strength by document type:")
print(
    doc_level_df.groupby('doc_type')['obligation_strength_score']
    .agg(['mean','std','min','max'])
    .round(3)
)


# Domain co-occurrence with obligation levels


ob_sents = sentences_df[
    sentences_df['obligation_level'].isin(['strong','moderate','weak'])
]

domain_cols = [c for c in sentences_df.columns if c.startswith('domain_')]

print("\nDomain association with obligation levels:")

results = []

for level in ['strong', 'moderate', 'weak']:
    level_sents = ob_sents[ob_sents['obligation_level'] == level]
    n = len(level_sents)

    print(f"\n{level.upper()} obligation (n={n:,} sentences):")

    for col in domain_cols:
        rate = level_sents[col].mean()

        results.append({
            "obligation_level": level,
            "domain": col.replace("domain_",""),
            "rate": rate
        })

        bar = "█" * int(rate * 30)
        print(f"  {col.replace('domain_',''):<30} {rate:.3f}  {bar}")

# Save structured version for later analysis
domain_obligation_df = pd.DataFrame(results)
domain_obligation_df.to_csv("domain_obligation_relationships.csv", index=False)

DOMAIN–OBLIGATION ANALYSIS

Obligation strength by company:
                 mean    std    min    max
company                                   
Anthropic       0.326  0.160  0.000  0.545
GoogleDeepMind  0.217  0.146  0.000  0.500
Meta            0.263  0.089  0.093  0.400
Microsoft       0.262  0.217  0.000  0.529
OpenAI          0.231  0.135  0.000  0.434

Obligation strength by document type:
                      mean    std    min    max
doc_type                                       
assessment           0.216    NaN  0.216  0.216
audit_report         0.361    NaN  0.361  0.361
gov_response         0.241  0.030  0.220  0.262
model_card           0.295  0.161  0.000  0.500
policy_governance    0.263  0.178  0.000  0.545
safety_report        0.275    NaN  0.275  0.275
system_card          0.247  0.145  0.036  0.434
technical_report     0.166  0.121  0.000  0.293
transparency_report  0.283  0.162  0.000  0.529

Domain association with obligation levels:

STRONG obligation (n=876 sentences):
  safety_risk                    0.341  ██████████
  fairness_bias                  0.038  █
  transparency_disclosure        0.056  █
  privacy_data_governance        0.022  
  governance_accountability      0.102  ███
  regulatory_compliance          0.185  █████

MODERATE obligation (n=787 sentences):
  safety_risk                    0.353  ██████████
  fairness_bias                  0.056  █
  transparency_disclosure        0.097  ██
  privacy_data_governance        0.034  █
  governance_accountability      0.113  ███
  regulatory_compliance          0.131  ███

WEAK obligation (n=1,304 sentences):
  safety_risk                    0.396  ███████████
  fairness_bias                  0.064  █
  transparency_disclosure        0.058  █
  privacy_data_governance        0.032  
  governance_accountability      0.080  ██
  regulatory_compliance          0.093  ██

# Domains with LOW strong obligation rates

strong_df = domain_obligation_df[
    domain_obligation_df['obligation_level'] == 'strong'
]

print("\nDomains least associated with strong obligation:")

print(
    strong_df.sort_values('rate')
    .head(10)
)


Domains least associated with strong obligation:
  obligation_level                     domain      rate
3           strong    privacy_data_governance  0.021689
1           strong              fairness_bias  0.037671
2           strong    transparency_disclosure  0.055936
4           strong  governance_accountability  0.101598
5           strong      regulatory_compliance  0.184932
0           strong                safety_risk  0.341324

# Step 9: Enforcement Gap analysis in addition to the above

# This analysis compares how often each domain appears in:
#   - strong obligation sentences
#   - weak obligation sentences
#
# A positive gap means the domain appears more in weak than strong
# obligation, suggesting it is framed more aspirationally than enforceably.

strong_sents = ob_sents[ob_sents['obligation_level'] == 'strong']
weak_sents   = ob_sents[ob_sents['obligation_level'] == 'weak']

gaps = []

for col in domain_cols:
    domain = col.replace('domain_', '')

    # Safely compute rates
    rate_str  = strong_sents[col].mean() if len(strong_sents) > 0 else 0
    rate_weak = weak_sents[col].mean()   if len(weak_sents)   > 0 else 0

    gap = rate_weak - rate_str

    gaps.append({
        'domain': domain,
        'strong_rate': round(rate_str, 3),
        'weak_rate': round(rate_weak, 3),
        'gap': round(gap, 3)
    })

gap_df = pd.DataFrame(gaps).sort_values('gap', ascending=False)

#ranking the above
gap_df['category'] = gap_df['gap'].apply(
    lambda x: 'aspirational' if x > 0 else 'enforced'
)

print("ENFORCEMENT GAP ANALYSIS")
print("Positive gap = domain appears more in weak than strong obligation")
print("Higher values indicate more aspirational framing\n")

print(gap_df)

# Save for results and figures
gap_df.to_csv("enforcement_gap.csv", index=False)

ENFORCEMENT GAP ANALYSIS
Positive gap = domain appears more in weak than strong obligation
Higher values indicate more aspirational framing

                      domain  strong_rate  weak_rate    gap      category
0                safety_risk        0.341      0.396  0.055  aspirational
1              fairness_bias        0.038      0.064  0.027  aspirational
3    privacy_data_governance        0.022      0.032  0.011  aspirational
2    transparency_disclosure        0.056      0.058  0.002  aspirational
4  governance_accountability        0.102      0.080 -0.022      enforced
5      regulatory_compliance        0.185      0.093 -0.092      enforced

# Step 10: Pointwise Mutual Information (PMI)
#
# PMI measures whether a domain and obligation level co-occur
# more or less often than expected under independence.
#
# Positive PMI  → stronger-than-expected association
# Negative PMI  → weaker-than-expected association

def compute_pmi(sentences_df, domain_cols):

    n_total = len(sentences_df)
    results = []

    for level in ['strong', 'moderate', 'weak']:

        # Probability of obligation level
        p_level = (sentences_df['obligation_level'] == level).mean()

        for col in domain_cols:
            domain = col.replace('domain_', '')

            # Probability of domain
            p_domain = sentences_df[col].mean()

            # Joint probability
            p_joint = (
                (sentences_df['obligation_level'] == level) &
                (sentences_df[col] == 1)
            ).mean()

            # PMI calculation
            if p_joint > 0 and p_level > 0 and p_domain > 0:
                pmi = np.log2(p_joint / (p_level * p_domain))
            else:
                pmi = np.nan

            results.append({
                'obligation_level': level,
                'domain': domain,
                'p_joint': round(p_joint, 4),
                'pmi': round(pmi, 3) if not np.isnan(pmi) else np.nan
            })

    return pd.DataFrame(results)


pmi_df = compute_pmi(sentences_df, domain_cols)

# Just better interpretation labels
#pmi_df['association'] = pmi_df['pmi'].apply(
 #   lambda x: 'strong association' if x > 0 else 'weak association'
#)

def label_pmi(x):
    if pd.isna(x):
        return "undefined"
    elif x > 0.5:
        return "strong positive"
    elif x > 0:
        return "positive"
    elif x < -0.5:
        return "strong negative"
    elif x < 0:
        return "negative"
    else:
        return "neutral"

pmi_df['association'] = pmi_df['pmi'].apply(label_pmi)


print("PMI RESULTS")

for level in ['strong', 'moderate', 'weak']:
    subset = pmi_df[pmi_df['obligation_level'] == level].sort_values(
        'pmi', ascending=False
    )
    
    print(f"\n{level.upper()} obligation:")
    print(subset[['domain','pmi','p_joint','association']])

# Save results
pmi_df.to_csv("pmi_analysis.csv", index=False)

PMI RESULTS

STRONG obligation:
                      domain    pmi  p_joint      association
5      regulatory_compliance  1.339   0.0062  strong positive
4  governance_accountability  0.443   0.0034         positive
0                safety_risk  0.426   0.0114         positive
3    privacy_data_governance  0.243   0.0007         positive
1              fairness_bias -0.010   0.0013         negative
2    transparency_disclosure -0.154   0.0019         negative

MODERATE obligation:
                       domain    pmi  p_joint      association
9     privacy_data_governance  0.905   0.0010  strong positive
11      regulatory_compliance  0.840   0.0039  strong positive
8     transparency_disclosure  0.634   0.0029  strong positive
10  governance_accountability  0.598   0.0034  strong positive
7               fairness_bias  0.559   0.0017  strong positive
6                 safety_risk  0.476   0.0106         positive

WEAK obligation:
                       domain    pmi  p_joint      association
15    privacy_data_governance  0.814   0.0016  strong positive
13              fairness_bias  0.764   0.0032  strong positive
12                safety_risk  0.643   0.0197  strong positive
17      regulatory_compliance  0.344   0.0046         positive
16  governance_accountability  0.094   0.0040         positive
14    transparency_disclosure -0.114   0.0029         negative

# Step 11: RQ4 Regression- Predicting Obligation Strength

# Outcome: obligation_strength
# Predictors:
#   - domain variables
#   - company
#   - document type
#   - year
#   - document length

print("RQ4 REGRESSION — Predicting obligation strength")


# Prepare regression data


reg_df = doc_level_df.copy()

# Standardize word count (helps model stability)
reg_df['word_count_std'] = (
    (reg_df['word_count'] - reg_df['word_count'].mean()) /
    reg_df['word_count'].std()
)

# Domain predictors (ALL domains, not just safety)
domain_vars = [c for c in reg_df.columns if c.startswith('prop_domain_')]

# Convert categorical variables to dummies
reg_df = pd.get_dummies(reg_df, columns=['company','doc_type'], drop_first=True)

company_dummies = [c for c in reg_df.columns if c.startswith('company_')]
doctype_dummies = [c for c in reg_df.columns if c.startswith('doc_type_')]

# Base predictors
predictors = domain_vars + company_dummies + doctype_dummies + ['year','word_count_std']


# Optional interaction term


if 'doc_type_system_card' in reg_df.columns:
    reg_df['safety_x_systemcard'] = (
        reg_df['prop_domain_safety_risk'] * reg_df['doc_type_system_card']
    )
    predictors.append('safety_x_systemcard')


# Run regression

y = reg_df['obligation_strength_score']

X = reg_df[predictors].astype(float)
X = sm.add_constant(X)

# Drop any rows with missing values in either y or X
valid_idx = y.notna() & X.notna().all(axis=1)

y_clean = y[valid_idx]
X_clean = X[valid_idx]

model = sm.OLS(y_clean, X_clean).fit(cov_type='HC3')

print(model.summary())

# Save output
with open("rq4_regression.txt", 'w') as f:
    f.write(model.summary().as_text())

print("Saved rq4_regression.txt")
print(model.params.sort_values(ascending=False))

RQ4 REGRESSION — Predicting obligation strength
                                OLS Regression Results                               
=====================================================================================
Dep. Variable:     obligation_strength_score   R-squared:                       0.414
Model:                                   OLS   Adj. R-squared:                  0.072
Method:                        Least Squares   F-statistic:                     38.48
Date:                       Mon, 06 Apr 2026   Prob (F-statistic):           1.17e-18
Time:                               22:13:02   Log-Likelihood:                 43.301
No. Observations:                         58   AIC:                            -42.60
Df Residuals:                             36   BIC:                             2.728
Df Model:                                 21                                         
Covariance Type:                         HC3                                         
=========================================================================================================
                                            coef    std err          z      P>|z|      [0.025      0.975]
---------------------------------------------------------------------------------------------------------
const                                   -25.7802     87.306     -0.295      0.768    -196.896     145.335
prop_domain_safety_risk                   0.0535      0.286      0.187      0.852      -0.508       0.615
prop_domain_fairness_bias                -0.7261      1.349     -0.538      0.590      -3.371       1.918
prop_domain_transparency_disclosure       0.5935      0.819      0.725      0.469      -1.012       2.199
prop_domain_privacy_data_governance       0.0604      2.392      0.025      0.980      -4.628       4.749
prop_domain_governance_accountability     0.3384      1.129      0.300      0.764      -1.874       2.551
prop_domain_regulatory_compliance         0.4732      0.679      0.697      0.486      -0.857       1.803
company_GoogleDeepMind                   -0.1045      0.091     -1.151      0.250      -0.282       0.073
company_Meta                             -0.0839      0.127     -0.660      0.509      -0.333       0.165
company_Microsoft                        -0.0687      0.139     -0.495      0.621      -0.341       0.203
company_OpenAI                           -0.0377      0.105     -0.359      0.720      -0.244       0.168
doc_type_audit_report                     0.0422      0.487      0.087      0.931      -0.912       0.997
doc_type_gov_response                    -0.0155      0.245     -0.063      0.949      -0.495       0.464
doc_type_model_card                       0.2116      0.341      0.620      0.535      -0.457       0.880
doc_type_policy_governance                0.0367      0.325      0.113      0.910      -0.600       0.673
doc_type_safety_report                   -0.0942      0.410     -0.230      0.818      -0.897       0.709
doc_type_system_card                     -0.1511      0.375     -0.402      0.687      -0.887       0.585
doc_type_technical_report                 0.0031      0.326      0.009      0.992      -0.636       0.642
doc_type_transparency_report              0.1117      0.322      0.347      0.729      -0.519       0.743
year                                      0.0128      0.043      0.296      0.767      -0.072       0.097
word_count_std                            0.0892      0.037      2.438      0.015       0.017       0.161
safety_x_systemcard                       0.7637      0.718      1.063      0.288      -0.644       2.172
==============================================================================
Omnibus:                        1.438   Durbin-Watson:                   2.251
Prob(Omnibus):                  0.487   Jarque-Bera (JB):                1.154
Skew:                          -0.119   Prob(JB):                        0.562
Kurtosis:                       2.351   Cond. No.                     7.91e+06
==============================================================================

Notes:
[1] Standard Errors are heteroscedasticity robust (HC3)
[2] The condition number is large, 7.91e+06. This might indicate that there are
strong multicollinearity or other numerical problems.
Saved rq4_regression.txt
safety_x_systemcard                       0.763665
prop_domain_transparency_disclosure       0.593463
prop_domain_regulatory_compliance         0.473224
prop_domain_governance_accountability     0.338391
doc_type_model_card                       0.211607
doc_type_transparency_report              0.111724
word_count_std                            0.089185
prop_domain_privacy_data_governance       0.060436
prop_domain_safety_risk                   0.053519
doc_type_audit_report                     0.042228
doc_type_policy_governance                0.036706
year                                      0.012794
doc_type_technical_report                 0.003084
doc_type_gov_response                    -0.015493
company_OpenAI                           -0.037736
company_Microsoft                        -0.068651
company_Meta                             -0.083859
doc_type_safety_report                   -0.094151
company_GoogleDeepMind                   -0.104485
doc_type_system_card                     -0.151054
prop_domain_fairness_bias                -0.726068
const                                   -25.780211
dtype: float64

# STEP 12: Exploratory — Within Company Consistency

# Measures how similar a company's documents are to each other.
# Here, I am representing each document as a vector of:
#   - obligation proportions
#   - domain proportions
# Then compute pairwise cosine similarity within each company.

def cosine_sim(a, b):
    # Add a small epsilon to avoid division by zero
    denom = (np.linalg.norm(a) * np.linalg.norm(b)) + 1e-12
    return np.dot(a, b) / denom


# Feature vector for each document
feature_cols = (
    ['prop_strong_obligation', 'prop_moderate_obligation', 'prop_weak_obligation'] +
    [c for c in doc_level_df.columns if c.startswith('prop_domain_')]
)

consistency = []

for company in doc_level_df['company'].unique():
    co_docs = doc_level_df[doc_level_df['company'] == company]

    # Need at least 2 documents to compare
    if len(co_docs) < 2:
        continue

    vectors = co_docs[feature_cols].fillna(0).values
    sims = []

    for i in range(len(vectors) - 1):
        for j in range(i + 1, len(vectors)):
            sims.append(cosine_sim(vectors[i], vectors[j]))

    consistency.append({
        'company': company,
        'mean_similarity': round(np.mean(sims), 3),
        'std_similarity': round(np.std(sims), 3),
        'n_documents': len(co_docs)
    })

consistency_df = pd.DataFrame(consistency).sort_values(
    'mean_similarity', ascending=False
)

print("WITHIN-COMPANY DISCOURSE CONSISTENCY")
print("Higher values indicate more consistent thematic focus across documents\n")
print(consistency_df)

# Save results
consistency_df.to_csv("within_company_consistency.csv", index=False)

WITHIN-COMPANY DISCOURSE CONSISTENCY
Higher values indicate more consistent thematic focus across documents

          company  mean_similarity  std_similarity  n_documents
1          OpenAI            0.956           0.035           12
2  GoogleDeepMind            0.910           0.079           16
0       Anthropic            0.897           0.084           14
4            Meta            0.802           0.148            9
3       Microsoft            0.793           0.140            7

# Step 14: Figures for the paper

# Global style for all figures

plt.style.use('default')

plt.rcParams.update({
    'font.family': 'serif',
    'font.size': 11,
    'axes.spines.top': False,
    'axes.spines.right': False,
    'axes.titleweight': 'bold',
    'axes.labelsize': 11,
    'axes.titlesize': 13,
    'legend.frameon': False
})

# Consistent color palette
COLORS = {
    'Anthropic': '#4C78A8',
    'OpenAI': '#F58518',
    'GoogleDeepMind': '#54A24B',
    'Microsoft': '#B279A2',
    'Meta': '#E45756'
}

NEUTRAL = '#4A4A4A'
LIGHT_GRAY = '#B0B0B0'


# Figure 1: Obligation strength by company
fig, ax = plt.subplots(figsize=(10, 5))

co_data = doc_level_df.groupby('company')['obligation_strength_score'].agg(['mean','std'])
co_data = co_data.reindex([c for c in COLORS.keys() if c in co_data.index])

colors = [COLORS[c] for c in co_data.index]

ax.bar(range(len(co_data)), co_data['mean'],
       yerr=co_data['std'],
       capsize=5,
       color=colors,
       edgecolor='white')

ax.set_xticks(range(len(co_data)))
ax.set_xticklabels(co_data.index, rotation=15, ha='right')

ax.set_ylabel("Obligation strength score")
ax.set_title("Obligation strength by company")

ax.set_ylim(0, 1)

# Reference line
ax.axhline(doc_level_df['obligation_strength_score'].mean(),
           color=LIGHT_GRAY, linestyle='--', linewidth=1)

plt.tight_layout()
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig1_obligation_by_company.png"), dpi=300)
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig1_obligation_by_company.pdf"))

plt.show()


# Figure 2: Obligation strength by document type
fig, ax = plt.subplots(figsize=(10, 5))

dt_data = doc_level_df.groupby('doc_type')['obligation_strength_score'].agg(['mean','std'])
dt_data = dt_data.sort_values('mean')

ax.barh(range(len(dt_data)), dt_data['mean'],
        xerr=dt_data['std'],
        capsize=4,
        color='#6BAED6',  # soft blue
        edgecolor='white')

ax.set_yticks(range(len(dt_data)))
ax.set_yticklabels([t.replace('_',' ').title() for t in dt_data.index])

ax.set_xlabel("Obligation strength score")
ax.set_title("Obligation strength by document type")

ax.axvline(doc_level_df['obligation_strength_score'].mean(),
           color=LIGHT_GRAY, linestyle='--', linewidth=1)

plt.tight_layout()
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig2_obligation_by_doctype.png"), dpi=300)
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig2_obligation_by_doctype.pdf"))

plt.show()


# Figure 3: Actor responsibility heatmap by company
#Commenting out because I do not need thid 
'''
fig, ax = plt.subplots(figsize=(10, 6))

actor_data = doc_level_df.groupby('company')[
    ['prop_actor_organization','prop_actor_developer',
     'prop_actor_user','prop_actor_ai_system']
].mean()

actor_data.columns = ['Organization','Developer','User','AI System']

sns.heatmap(
s   actor_data,
    annot=True,
    fmt='.2f',
    cmap='Blues',
    linewidths=0.5,
    cbar_kws={'label':'Proportion'},
    ax=ax
)

ax.set_title("Actor responsibility distribution by company")
ax.set_xlabel("")
ax.set_ylabel("")

plt.tight_layout()
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig3_actor_heatmap.png"), dpi=300)
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig3_actor_heatmap.pdf"))

plt.show()

'''

# Figure 4: Enforcement gap — what stays aspirational?
fig, ax = plt.subplots(figsize=(10, 6))

ep = gap_df.sort_values('gap')

colors = ['#D55E00' if g > 0 else '#0072B2' for g in ep['gap']]

ax.barh(range(len(ep)), ep['gap'],
        color=colors,
        edgecolor='white')

ax.set_yticks(range(len(ep)))
ax.set_yticklabels([d.replace('_',' ').title() for d in ep['domain']])

ax.set_xlabel("Enforcement gap (weak minus strong)")
ax.set_title("Enforcement gap across domains")

ax.axvline(0, color='black', linewidth=0.8)

plt.tight_layout()
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig4_enforcement_gap.png"), dpi=300)
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig4_enforcement_gap.pdf"))

plt.show()


# Figure 5: Obligation strength over time by company
fig, ax = plt.subplots(figsize=(10, 5))

time_data = doc_level_df.groupby(['year','company'])['obligation_strength_score'].mean().reset_index()

for company in time_data['company'].unique():
    ct = time_data[time_data['company'] == company]

    if len(ct) > 1:
        ax.plot(
            ct['year'],
            ct['obligation_strength_score'],
            marker='o',
            label=company,
            linewidth=2,
            color=COLORS.get(company, NEUTRAL)
        )

ax.set_xlabel("Year")
ax.set_ylabel("Obligation strength score")
ax.set_title("Obligation strength over time")

ax.legend(fontsize=9)

plt.tight_layout()
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig5_obligation_over_time.png"), dpi=300)
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig5_obligation_over_time.pdf"))

plt.show()

#Domain-Obligation Heatmap
fig, ax = plt.subplots(figsize=(10, 6))

heat_data = domain_obligation_df.pivot(
    index='domain',
    columns='obligation_level',
    values='rate'
)

sns.heatmap(
    heat_data,
    annot=True,
    fmt=".2f",
    cmap="Blues",
    linewidths=0.5,
    cbar_kws={'label': 'Proportion'},
    ax=ax
)

ax.set_title("Domain–Obligation Relationship")
ax.set_xlabel("Obligation Level")
ax.set_ylabel("Domain")

plt.tight_layout()
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig_domain_obligation_heatmap.png"), dpi=300)
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig_domain_obligation_heatmap.pdf"))

plt.show()

fig, ax = plt.subplots(figsize=(10, 6))

pmi_heat = pmi_df.pivot(
    index='domain',
    columns='obligation_level',
    values='pmi'
)

sns.heatmap(
    pmi_heat,
    annot=True,
    cmap="coolwarm",
    center=0,
    fmt=".2f",
    linewidths=0.5,
    cbar_kws={'label': 'PMI'},
    ax=ax
)

ax.set_title("PMI: Domain–Obligation Associations")

plt.tight_layout()
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig_pmi_heatmap.png"), dpi=300)
plt.savefig(os.path.join(OUTPUT_FOLDER, "fig_pmi_heatmap.pdf"))
plt.show()

print("Outputs saved to /kaggle/working/")

print("\nKey files:")
print("- sentences_analyzed.csv")
print("- document_level_stats.csv")
print("- enforcement_gap.csv")
print("- pmi_analysis.csv")
print("- rq4_regression.txt")

print("\nFigures:")
print("- fig1 to fig5")

print("\nNext step: Run R notebook for STM (topic modeling)")

Outputs saved to /kaggle/working/

Key files:
- sentences_analyzed.csv
- document_level_stats.csv
- enforcement_gap.csv
- pmi_analysis.csv
- rq4_regression.txt

Figures:
- fig1 to fig5

Next step: Run R notebook for STM (topic modeling)

print(reg_df.columns.tolist())

['doc_id', 'n_sentences', 'doc_length_log', 'prop_strong_obligation', 'prop_moderate_obligation', 'prop_weak_obligation', 'prop_none_obligation', 'obligation_strength_score', 'n_obligation_sentences', 'prop_domain_safety_risk', 'prop_domain_fairness_bias', 'prop_domain_transparency_disclosure', 'prop_domain_privacy_data_governance', 'prop_domain_governance_accountability', 'prop_domain_regulatory_compliance', 'year', 'filename', 'word_count', 'word_count_std', 'company_GoogleDeepMind', 'company_Meta', 'company_Microsoft', 'company_OpenAI', 'doc_type_audit_report', 'doc_type_gov_response', 'doc_type_model_card', 'doc_type_policy_governance', 'doc_type_safety_report', 'doc_type_system_card', 'doc_type_technical_report', 'doc_type_transparency_report', 'safety_x_systemcard']