Esempi di codice

La stessa ricerca in diversi linguaggi: trovare i siti che contengono un certo script, sfogliare i risultati e fermarsi educatamente quando viene chiesto di aspettare. Metti la tua chiave in una variabile d'ambiente invece che nel file.

Le stesse cose, raccolte in piccoli client con riga di comando, solo con la libreria standard: Python, JavaScript, PHP, Go, Ruby. In un foglio di calcolo: Google Sheets. In una tabella di prospecting: Clay.

curl

export PUBLICWWW_KEY=...

# One page of results
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' --data 'per_page=20' \
     https://api.publicwww.com/v1/search

# Every match, as a plain list of urls
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' \
     --data 'format=txt&per_page=100000' \
     https://api.publicwww.com/v1/search > sites.txt

# A long, multi-line query is easier as a POST
curl https://api.publicwww.com/v1/search \
     -H "Authorization: Bearer $PUBLICWWW_KEY" \
     -H "Content-Type: application/json" \
     -d '{"query": ["\"angular.min.js\"", "\"bootstrap.min.css\""], "per_page": 50}'

Python

import os, time, requests

KEY  = os.environ["PUBLICWWW_KEY"]
BASE = "https://api.publicwww.com"

def search(query, **params):
    """One page of results, waiting out the rate limit if asked to."""
    while True:
        r = requests.get(BASE + "/v1/search",
                         headers={"Authorization": "Bearer " + KEY},
                         params={"query": query, **params})
        if r.status_code == 429:
            body = r.json()["error"]
            if body["code"] != "too_many_requests":
                raise RuntimeError(body["message"])       # quota, not pace
            time.sleep(int(r.headers.get("Retry-After", 30)))
            continue
        r.raise_for_status()
        return r.json()

page = search('"angular.min.js"', per_page=20)
print(page["total"], "sites match")
for row in page["results"]:
    print(row["rank"], row["domain"], row["url"])

Ricevere in streaming un grande insieme di risultati

import json, os, requests

r = requests.get("https://api.publicwww.com/v1/search",
                 headers={"Authorization": "Bearer " + os.environ["PUBLICWWW_KEY"]},
                 params={"query": '"angular.min.js"',
                         "format": "ndjson", "per_page": 1000000},
                 stream=True)
r.raise_for_status()

meta = None
for line in r.iter_lines():
    if not line:
        continue
    obj = json.loads(line)
    if obj.get("object") == "meta":          # always the first line
        meta = obj
        print("expecting", meta["returned"], "of", meta["total"])
        continue
    print(obj["domain"])

PHP

<?php
$key = getenv ("PUBLICWWW_KEY");

$url = "https://api.publicwww.com/v1/search?" . http_build_query (array (
	"query"    => '"angular.min.js"',
	"per_page" => 20,
));

$ch = curl_init ($url);
curl_setopt ($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt ($ch, CURLOPT_HTTPHEADER, array ("Authorization: Bearer " . $key));

$body   = curl_exec ($ch);
$status = curl_getinfo ($ch, CURLINFO_HTTP_CODE);
curl_close ($ch);

$data = json_decode ($body, true);

if ($status != 200) {
	die ($data ["error"] ["code"] . ": " . $data ["error"] ["message"] . "\n");
}

echo $data ["total"] . " sites match\n";
foreach ($data ["results"] as $r) {
	echo $r ["rank"] . "\t" . $r ["domain"] . "\t" . $r ["url"] . "\n";
}

JavaScript (Node)

const KEY  = process.env.PUBLICWWW_KEY;
const BASE = "https://api.publicwww.com";

async function search (query, params = {}) {
  const url = new URL (BASE + "/v1/search");
  url.searchParams.set ("query", query);
  for (const [k, v] of Object.entries (params)) url.searchParams.set (k, v);

  for (;;) {
    const res  = await fetch (url, { headers: { Authorization: `Bearer ${KEY}` } });
    const body = await res.json ();

    if (res.status === 429 && body.error.code === "too_many_requests") {
      await new Promise (r => setTimeout (r, body.error.retry_after * 1000));
      continue;                                   // told to wait, not refused
    }
    if (!res.ok) throw new Error (`${body.error.code}: ${body.error.message}`);
    return body;
  }
}

const page = await search ('"angular.min.js"', { per_page: 20 });
console.log (page.total, "sites match");
for (const r of page.results) console.log (r.rank, r.domain, r.url);

Go

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
)

type Result struct {
	Domain string `json:"domain"`
	URL    string `json:"url"`
	Rank   *int   `json:"rank"`          // nil when the site has no rank
}

type Page struct {
	Total   int      `json:"total"`
	Results []Result `json:"results"`
}

func main () {
	q := url.Values{}
	q.Set ("query", `"angular.min.js"`)
	q.Set ("per_page", "20")

	req, _ := http.NewRequest ("GET", "https://api.publicwww.com/v1/search?"+q.Encode (), nil)
	req.Header.Set ("Authorization", "Bearer "+os.Getenv ("PUBLICWWW_KEY"))

	res, err := http.DefaultClient.Do (req)
	if err != nil { panic (err) }
	defer res.Body.Close ()

	if res.StatusCode != 200 {
		fmt.Println ("http", res.StatusCode, res.Header.Get ("Retry-After"))
		os.Exit (1)
	}

	var page Page
	json.NewDecoder (res.Body).Decode (&page)

	fmt.Println (page.Total, "sites match")
	for _, r := range page.Results {
		fmt.Println (r.Domain, r.URL)
	}
}

Ruby

require "json"
require "net/http"

KEY = ENV.fetch ("PUBLICWWW_KEY")

uri = URI ("https://api.publicwww.com/v1/search")
uri.query = URI.encode_www_form (query: '"angular.min.js"', per_page: 20)

req = Net::HTTP::Get.new (uri)
req["Authorization"] = "Bearer #{KEY}"

res = Net::HTTP.start (uri.host, uri.port, use_ssl: true) { |http| http.request (req) }
body = JSON.parse (res.body)

abort "#{body['error']['code']}: #{body['error']['message']}" unless res.code == "200"

puts "#{body['total']} sites match"
body["results"].each { |r| puts [r["rank"], r["domain"], r["url"]].join ("\t") }

Caricare i dati in un foglio di calcolo o in una pipeline della shell

# A csv of domain and rank, ready for a spreadsheet
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' \
     --data 'format=csv&header=1&delimiter=,&per_page=10000' \
     https://api.publicwww.com/v1/search > sites.csv

# Just the domains, one per line, into whatever comes next
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' \
     --data 'format=csv&columns=domain&per_page=10000' \
     https://api.publicwww.com/v1/search | sort -u

Cose da fare bene

Distingui i due 429. too_many_requests significa rallenta e ripeti. quota_exceeded significa torna domani; riprovare in un ciclo non serve e sembrerà un attacco.
Rispetta Retry-After invece di indovinare un intervallo. È il numero esatto di secondi prima che si liberi uno slot.
Controlla truncated prima di considerare completo un insieme di risultati.
Aspettati rank: null. Molti siti indicizzati non hanno una posizione in classifica; un linguaggio con interi non annullabili richiede un puntatore o un tipo opzionale, come mostra l'esempio in Go.
Sfoglia con page invece di rieseguire varianti della query per raggiungere le righe successive.

Successivo Google Sheets