darwin PMDA: Enhanced metrics for memory, VFS, and network subsystems

.cirrus.yml

@@ -0,0 +1,81 @@
+task:
+  name: macOS PCP Build
+  macos_instance:
+    image: ghcr.io/cirruslabs/macos-tahoe-base:latest
+  # Cache Homebrew prefix + download cache
+  homebrew_cache:
+    folder: /opt/homebrew
+    # Also cache downloaded bottles to avoid re-downloading on cache miss/rebuild
+    # (you can list multiple folders)
+    # folders:
+    #   - /opt/homebrew
+    #   - ~/Library/Caches/Homebrew
+    fingerprint_script: |
+      brew --version
+      echo $HOMEBREW_NO_AUTO_UPDATE  # if you use it
+      brew list --versions | sort    # list installed formulae & versions
+    populate_script: |
+      # This runs only on cache miss — safe to install everything here
+      echo "Populating Homebrew cache..."
+      # Homebrew is already in base images, but we can ensure update if needed
+      brew update --quiet || true
+      brew install coreutils gnu-tar pkg-config python3 python-setuptools autoconf || true  # idempotent
+    # Optional: clean up non-deterministic files before cache upload
+    before_cache_script: |
+      brew cleanup --prune=all || true
+      rm -rf ~/Library/Caches/Homebrew/downloads  # optional, if too noisy
+  #brew_setup_script:
+    # Now this will be very fast if cache hit
+    #- echo "Setting up dependencies (cached!)"
+#    - brew install coreutils gnu-tar pkg-config python3 python-setuptools autoconf || true  # idempotent
+  enable_tcp_stats_script: |
+    echo "Enabling TCP statistics for testing..."
+    sudo sysctl -w net.inet.tcp.disable_access_to_stats=0
+    echo "Verifying setting:"
+    sysctl net.inet.tcp.disable_access_to_stats
+  pcp_build_script: |
+     [ "$PCP_SKIP_BUILD" == "true" ] && echo "SKIPPING BUILD" || ./Makepkgs --verbose
+  pcp_install_script: |
+     DMG_PATH=$(find pcp-**/build/mac -name "pcp-*.dmg" | head -1) # Find the generated DMG
+     echo "Found DMG: $DMG_PATH"
+     MOUNT_OUTPUT=$(hdiutil attach "$DMG_PATH" | tail -1) # Mount the DMG
+     VOLUME_PATH=$(echo "$MOUNT_OUTPUT" | awk '{print $3}')
+     echo "Mounted at: $VOLUME_PATH"
+     PKG_PATH=$(find "$VOLUME_PATH" -name "*.pkg" | head -1) # Find and install the PKG
+     echo "Found PKG: $PKG_PATH"
+     ls -alh $PKG_PATH
+     pkgutil --check-signature $PKG_PATH || true # this returns 1 when things are ok
+     sudo installer -pkg "$PKG_PATH" -target / -verbose # Install (sudo is passwordless in GitHub Actions)
+  check_pmcd_is_running_postinstall_script: |
+    echo "Waiting for pcp service to start (it can take a while after install)..."
+    TIMEOUT=180  # 3 minutes
+    ELAPSED=0
+    INTERVAL=5
+
+    while [ $ELAPSED -lt $TIMEOUT ]; do
+      if pcp 2>/dev/null; then
+        echo "pcp service is responding"
+        break
+      fi
+      sleep $INTERVAL
+      ELAPSED=$((ELAPSED + INTERVAL))
+    done
+
+    if [ $ELAPSED -ge $TIMEOUT ]; then
+      echo "ERROR: pcp service failed to start within $TIMEOUT seconds"
+      exit 1
+    fi
+  run_unit_tests_script: |
+    cd scripts/darwin/test/unit
+    ./run-unit-tests.sh
+  run_integration_tests_script: |
+    cd scripts/darwin/test/integration
+    ./run-integration-tests.sh
+  pause_script: |
+    if [ "$PCP_PAUSE_AFTER_INSTALL" == "true" ]; then
+      echo "VM IP: $(ipconfig getifaddr en0)"
+      sleep 3600  # Keeps VM alive for an hour
+    else
+      echo "No pause required, finishing"
+    fi
+

.github/workflows/macOS.yml

@@ -31,12 +31,32 @@ jobs:
         id: install-brew-deps
         run: |
           brew install autoconf unixodbc
+      - name: Configure PCP
+        id: configure
+        run: |
+          echo "Configuring PCP..."
+          ./configure --prefix=/usr --libexecdir=/usr/lib --sysconfdir=/etc --localstatedir=/var
       - name: Build on macOS
         id: Build
         run: |
           source pybuilddeps/bin/activate
           xcodebuild -version
           ./Makepkgs -verbose
+      # Darwin PMDA Unit Tests (pre-installation)
+      - name: Run Darwin PMDA Unit Tests
+        id: unit-tests
+        if: success()
+        run: |
+          echo "Running Darwin PMDA unit tests..."
+          cd scripts/darwin/test/unit
+          ./run-unit-tests.sh
+      - name: Debug on unit test failure
+        if: failure() && steps.unit-tests.conclusion == 'failure'
+        run: |
+          echo "Unit tests failed. Checking build artifacts..."
+          find . -name "pmda_darwin.dylib" -o -name "pmdadarwin"
+          echo "Checking for pmcd build..."
+          find . -name "dbpmda"
       - name: Debug on failure
         if: failure() && steps.Build.conclusion == 'failure'
         run: |
@@ -94,6 +114,216 @@ jobs:
           echo "Checking for expected directories..."
           test -d /etc/pcp && echo "✓ /etc/pcp exists"
           test -d /var/lib/pcp && echo "✓ /var/lib/pcp exists"
+      # Darwin PMDA Integration Tests (post-installation)
+      - name: Wait for pmcd to start
+        if: success() && steps.install-test.conclusion == 'success'
+        id: wait-for-pmcd
+        run: |
+          echo "=== Installation Verification ==="
+          echo "PCP installation check:"
+          which pmcd && echo "pmcd binary: $(which pmcd)" || echo "pmcd not in PATH"
+          which pminfo && echo "pminfo binary: $(which pminfo)" || echo "pminfo not in PATH"
+          which pcp && echo "pcp binary: $(which pcp)" || echo "pcp not in PATH"
+
+          echo ""
+          echo "Critical directories:"
+          test -d /etc/pcp && echo "✓ /etc/pcp exists" || echo "✗ /etc/pcp MISSING"
+          test -d /var/lib/pcp && echo "✓ /var/lib/pcp exists" || echo "✗ /var/lib/pcp MISSING"
+          test -d /var/log/pcp && echo "✓ /var/log/pcp exists" || echo "✗ /var/log/pcp MISSING"
+          test -d /var/log/pcp/pmcd && echo "✓ /var/log/pcp/pmcd exists" || echo "✗ /var/log/pcp/pmcd MISSING"
+
+          echo ""
+          echo "pmcd.conf contents:"
+          cat /etc/pcp/pmcd/pmcd.conf 2>/dev/null || echo "pmcd.conf not found"
+
+          echo ""
+          echo "Checking launchd plist:"
+          test -f /Library/LaunchDaemons/org.pcp.pmcd.plist && echo "✓ org.pcp.pmcd.plist exists" || echo "✗ org.pcp.pmcd.plist MISSING"
+
+          echo ""
+          echo "=== Waiting for pmcd service to start (post-install) ==="
+          echo "The installer should have started pmcd via launchd..."
+
+          TIMEOUT=60  # Wait up to 60 seconds
+          ELAPSED=0
+          INTERVAL=3
+
+          while [ $ELAPSED -lt $TIMEOUT ]; do
+            echo "Checking if pmcd is responding (${ELAPSED}s elapsed)..."
+
+            # Check if pcp command works (like in .cirrus.yml)
+            if pcp 2>/dev/null; then
+              echo "✓ pmcd service is responding!"
+              break
+            fi
+
+            # Also check if pminfo can connect
+            if pminfo -h localhost -f hinv.ncpu 2>/dev/null; then
+              echo "✓ pmcd is accepting connections!"
+              break
+            fi
+
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+
+          if [ $ELAPSED -ge $TIMEOUT ]; then
+            echo "✗ ERROR: pmcd service failed to start within $TIMEOUT seconds"
+
+            echo ""
+            echo "=== Diagnostic Information ==="
+            echo "pmcd process status:"
+            pgrep pmcd && echo "pmcd is running (PID: $(pgrep pmcd))" || echo "pmcd NOT running"
+
+            echo ""
+            echo "pmcd full process details:"
+            ps aux | grep pmcd | grep -v grep || echo "No pmcd process found"
+
+            echo ""
+            echo "pmcd command line:"
+            sudo ps -p $(pgrep pmcd) -o pid,ppid,user,command 2>/dev/null || echo "Cannot get process details"
+
+            echo ""
+            echo "pmcd network connections:"
+            sudo lsof -i -P | grep pmcd || echo "No network connections"
+
+            echo ""
+            echo "pmcd log (last 30 lines):"
+            sudo tail -30 /var/log/pcp/pmcd/pmcd.log 2>/dev/null || echo "No pmcd.log"
+
+            echo ""
+            echo "launchd plist stdout/stderr:"
+            sudo cat /var/log/pcp/pmcd/plist.stdout 2>/dev/null || echo "No plist.stdout"
+            sudo cat /var/log/pcp/pmcd/plist.stderr 2>/dev/null || echo "No plist.stderr"
+
+            exit 1
+          fi
+
+          echo ""
+          echo "=== pmcd Service Verification ==="
+          echo "pmcd process status:"
+          pgrep pmcd && echo "✓ pmcd is running (PID: $(pgrep pmcd))" || echo "✗ pmcd failed to start"
+
+          echo ""
+          echo "pmcd command line:"
+          sudo ps -p $(pgrep pmcd) -o pid,ppid,user,command || echo "Cannot get process details"
+
+          echo ""
+          echo "pmcd listening ports:"
+          sudo lsof -i -P | grep pmcd || echo "No network connections found for pmcd"
+
+          echo ""
+          echo "Test localhost connectivity to pmcd port (44321):"
+          nc -z localhost 44321 && echo "✓ Can connect to localhost:44321" || echo "✗ Cannot connect to localhost:44321"
+
+          echo ""
+          echo "Test 127.0.0.1 connectivity to pmcd port (44321):"
+          nc -z 127.0.0.1 44321 && echo "✓ Can connect to 127.0.0.1:44321" || echo "✗ Cannot connect to 127.0.0.1:44321"
+      - name: Run Darwin PMDA Integration Tests
+        id: integration-tests
+        if: success() && steps.wait-for-pmcd.conclusion == 'success'
+        run: |
+          echo "=== Python and pmrep debugging ==="
+          echo "pmrep location:"
+          which pmrep
+          head -1 $(which pmrep)
+
+          echo ""
+          echo "Python versions available:"
+          which python3 && python3 --version || echo "python3 not found"
+          which python && python --version || echo "python not found"
+
+          echo ""
+          echo "Python sys.path for pmrep's Python:"
+          $(head -1 $(which pmrep) | sed 's/#!//') -c "import sys; print('\n'.join(sys.path))"
+
+          echo ""
+          echo "Looking for pcp module:"
+          find /usr/local -name "pcp.py" -o -name "pcp" -type d 2>/dev/null | grep -i python || echo "No pcp module found in /usr/local"
+
+          echo ""
+          echo "Check if pcp module is installed:"
+          $(head -1 $(which pmrep) | sed 's/#!//') -c "import pcp; print('pcp module location:', pcp.__file__)" 2>&1 || echo "pcp module not importable"
+
+          echo ""
+          echo "Setting PYTHONPATH to include PCP module location..."
+          export PYTHONPATH="/usr/local/lib/python3.13/site-packages:${PYTHONPATH:-}"
+          echo "PYTHONPATH=${PYTHONPATH}"
+
+          echo ""
+          echo "Verify pcp module is now importable:"
+          $(head -1 $(which pmrep) | sed 's/#!//') -c "import pcp; print('✓ pcp module found at:', pcp.__file__)"
+
+          echo ""
+          echo "Running Darwin PMDA integration tests..."
+          cd scripts/darwin/test/integration
+          ./run-integration-tests.sh
+      - name: Test pmstat with Darwin metrics
+        if: success() && steps.integration-tests.conclusion == 'success'
+        run: |
+          echo "Testing pmstat output..."
+          cd scripts/darwin/test/integration
+          ./test-pmstat.sh
+      - name: Debug on integration test failure
+        if: failure() && (steps.integration-tests.conclusion == 'failure' || steps.wait-for-pmcd.conclusion == 'failure')
+        run: |
+          echo "Integration tests failed. Collecting diagnostics..."
+
+          echo "=== PCP Configuration ==="
+          pmconfig 2>&1 || echo "pmconfig failed"
+
+          echo ""
+          echo "=== pmcd.conf contents ==="
+          cat /etc/pcp/pmcd/pmcd.conf 2>/dev/null || echo "pmcd.conf not found"
+
+          echo ""
+          echo "=== pmcd status ==="
+          pgrep pmcd && echo "pmcd is running (PID: $(pgrep pmcd))" || echo "pmcd not running"
+
+          echo ""
+          echo "=== pmcd network connections ==="
+          sudo lsof -i -P | grep pmcd || echo "No network connections for pmcd"
+
+          echo ""
+          echo "=== All listening ports ==="
+          sudo lsof -i -P | grep LISTEN || echo "No listening ports found"
+
+          echo ""
+          echo "=== Connectivity tests ==="
+          nc -z localhost 44321 && echo "✓ localhost:44321 is reachable" || echo "✗ localhost:44321 is NOT reachable"
+          nc -z 127.0.0.1 44321 && echo "✓ 127.0.0.1:44321 is reachable" || echo "✗ 127.0.0.1:44321 is NOT reachable"
+
+          echo ""
+          echo "=== pmcd log (last 50 lines) ==="
+          sudo tail -50 /var/log/pcp/pmcd/pmcd.log 2>/dev/null || echo "No pmcd log found"
+
+          echo ""
+          echo "=== darwin PMDA log (last 50 lines) ==="
+          sudo tail -50 /var/log/pcp/pmcd/darwin.log 2>/dev/null || echo "No darwin PMDA log found"
+
+          echo ""
+          echo "=== darwin PMDA status ==="
+          pminfo -f pmcd.agent.status 2>/dev/null | grep darwin || echo "Darwin PMDA not loaded"
+
+          echo ""
+          echo "=== Available PMDAs ==="
+          ls -la /usr/local/lib/pcp/pmdas/ 2>/dev/null || ls -la /Library/PCP/pmdas/ 2>/dev/null || echo "PMDA directory not found"
+
+          echo ""
+          echo "=== Test simple metric with explicit localhost ==="
+          pminfo -h localhost -f hinv.ncpu 2>&1 || echo "pminfo with localhost failed"
+
+          echo ""
+          echo "=== Test simple metric with explicit 127.0.0.1 ==="
+          pminfo -h 127.0.0.1 -f hinv.ncpu 2>&1 || echo "pminfo with 127.0.0.1 failed"
+
+          echo ""
+          echo "=== Test simple metric with default host ==="
+          pminfo -f hinv.ncpu 2>&1 || echo "pminfo with default host failed"
+
+          echo ""
+          echo "=== PCP environment variables ==="
+          env | grep PCP || echo "No PCP env vars set"
       - name: Cleanup - Unmount DMG
         if: always()
         run: |

.gitignore

@@ -52,3 +52,6 @@ provisioning/osxsierra.legally.ok
 *.gdb_history
 *vgcore.*
 tmpfiles.init.setup 
+
+# Claude Code working files
+.claude/

CONTRIBUTING.md

@@ -167,3 +167,10 @@ keep the differences minimal in order to ease future updates of vendored code.
 
 **All bugs and CVEs of a vendored library in turn become responsibilities of the
 PCP maintainers as well** (and fixes must be pushed upstream).
+
+## Platform-Specific Development
+
+### macOS with Tart VMs
+
+macOS developers can build in isolated VMs for clean, reproducible builds that match the CI environment. See [MACOS_DEVELOPMENT.md](MACOS_DEVELOPMENT.md) for details.
+