Ruby SDK for AdsCrawl

gem install adscrawl

bundle add adscrawl

require "adscrawl"

client = AdsCrawl::Client.new
markdown = client.markdown(
  url: "https://www.adscrawl.net",
  waitUntil: "domcontentloaded"
)
puts markdown

require "adscrawl"

client = AdsCrawl::Client.new
html = client.html(url: "https://www.adscrawl.net")
article = client.article(url: "https://www.adscrawl.net")
puts [article["title"], article["textContent"]]

png = client.screenshot(
  url: "https://www.adscrawl.net",
  viewport: {width: 1440, height: 900},
  fullPage: true,
  waitUntil: "load"
)
File.binwrite("page.png", png)

require "adscrawl"

client = AdsCrawl::Client.new
server = ENV["ADSCRAWL_PROXY_SERVER"]
username = ENV["ADSCRAWL_PROXY_USERNAME"]
password = ENV["ADSCRAWL_PROXY_PASSWORD"]
raise "Set both proxy username and password, or neither." if username.nil? != password.nil?
raise "Set ADSCRAWL_PROXY_SERVER with proxy credentials." if server.nil? && (username || password)

routing = {countryCode: "GLOBAL"}
if server
  proxy = {server: server}
  proxy.merge!(username: username, password: password) if username && password
  routing = {proxy: proxy}
end

png = client.screenshot(
  **routing,
  url: "https://www.browserscan.net/",
  viewport: {width: 1440, height: 900},
  fullPage: true,
  waitUntil: "networkidle",
  timeoutMs: 60_000,
  userAgentMode: "random",
  userAgentOs: "windows",
  fingerprint: {
    webRtc: "forward", webGl: "random", webGpu: "random",
    webGlImage: "random", canvas: "random", audioContext: "random",
    clientRects: "random", speechVoices: "random", fonts: "random",
    hardware: "random", doNotTrack: "random"
  },
  timeout_ms: 75_000
)
File.binwrite("browserscan.png", png)

require "adscrawl"

client = AdsCrawl::Client.new
puts client.spa.templates["templates"]

result = client.spa.extract(
  url: "https://www.adscrawl.net",
  fields: {
    title: {source: "dom", selector: "h1", value: "text", required: true}
  }
)
puts result.dig("data", "title")

inspection = client.spa.inspect(url: "https://www.adscrawl.net")
puts inspection["candidates"]

require "adscrawl"

client = AdsCrawl::Client.new
session = client.cdp.create(
  idleTimeoutMs: 600_000,
  maxSessionMs: 3_600_000,
  browserSettings: {viewport: {width: 1440, height: 900}}
)
begin
  version = client.cdp.get_version(session)
  puts version["Browser"]
  # Connect a CDP-compatible Ruby library to session["cdpBaseUrl"].
ensure
  client.cdp.close(session["sessionId"])
end

require "adscrawl"

client = AdsCrawl::Client.new
launched = client.cloud_browsers.launch(
  proxy: {server: ENV.fetch("ADSCRAWL_PROXY_SERVER")},
  tabs: ["https://www.adscrawl.net"]
)
browser_id = launched["id"]
begin
  profile = client.cloud_browsers.get(browser_id)
  puts profile.dig("runtime", "status")
ensure
  client.cloud_browsers.stop(browser_id)
end

require "adscrawl"

client = AdsCrawl::Client.new(base_url: "https://api.adscrawl.net", timeout_ms: 90_000)
begin
  puts client.markdown(
    {url: "https://www.adscrawl.net", timeoutMs: 60_000},
    timeout_ms: 75_000
  )
rescue AdsCrawl::APIError => error
  puts [error.status, error.api_code, error.trace_id]
rescue AdsCrawl::TimeoutError
  puts "Inspect remote sessions before retrying."
end