about summary refs log tree commit diff stats
path: root/scrape.rb
blob: a28f4c5d02dd36db56c54d53eca281f29b2ed69a (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
require 'json'
require 'nokogiri'
require 'open-uri'
require 'yaml'

config = YAML.load(open(ARGV[0]))
usernames = config["usernames"]

achieves = usernames.map do |username|
  page = Nokogiri::HTML(open("https://steamcommunity.com/#{username}/games/?tab=all"))
  script = page.css(".responsive_page_template_content script").text[18..-1]
  data = JSON.parse(script[0..script.index(";\r\n\t\t")-1])
  ids = data.map { |d| d["appid"] }

  index = 0
  ids.map do |id|
    index += 1
    puts "#{username} - #{index}/#{ids.count}"

    achsp = Nokogiri::HTML(open("https://steamcommunity.com/#{username}/stats/#{id}/"))
    achsp.css(".achieveTxt .achieveUnlockTime + h3").map { |d| d.text }
  end
end.flatten

if File.exists?(config["achievements"])
  already = File.read(config["achievements"]).split("\n")
  all_achieves = achieves + already
else
  all_achieves = achieves
end

all_achieves.sort!
all_achieves.uniq!

if config.key? "blacklist"
  blacklist = File.read(config["blacklist"]).split("\n")
  all_achieves.reject! { |l| blacklist.include? l }
end

File.open(config["achievements"], "w") do |f|
  f << all_achieves.join("\n")
end