diff options
Diffstat (limited to 'scrape.rb')
-rw-r--r-- | scrape.rb | 38 |
1 files changed, 38 insertions, 0 deletions
diff --git a/scrape.rb b/scrape.rb new file mode 100644 index 0000000..5d62e89 --- /dev/null +++ b/scrape.rb | |||
@@ -0,0 +1,38 @@ | |||
1 | require 'json' | ||
2 | require 'nokogiri' | ||
3 | require 'open-uri' | ||
4 | require 'yaml' | ||
5 | |||
6 | config = YAML.load(open("config.yml")) | ||
7 | usernames = config["usernames"] | ||
8 | |||
9 | achieves = usernames.map do |username| | ||
10 | page = Nokogiri::HTML(open("https://steamcommunity.com/id/#{username}/games/?tab=all")) | ||
11 | script = page.css(".responsive_page_template_content script").text[18..-1] | ||
12 | data = JSON.parse(script[0..script.index(";\r\n\t\t")-1]) | ||
13 | ids = data.map { |d| d["appid"] } | ||
14 | |||
15 | ids.map do |id| | ||
16 | achsp = Nokogiri::HTML(open("https://steamcommunity.com/id/#{username}/stats/#{id}/")) | ||
17 | achsp.css(".achieveTxt .achieveUnlockTime + h3").map { |d| d.text } | ||
18 | end | ||
19 | end.flatten | ||
20 | |||
21 | if File.exists?("achieves.txt") | ||
22 | already = File.read("achieves.txt").split("\n") | ||
23 | all_achieves = achieves + already | ||
24 | else | ||
25 | all_achieves = achieves | ||
26 | end | ||
27 | |||
28 | all_achieves.sort! | ||
29 | all_achieves.uniq! | ||
30 | |||
31 | if config.key? "blacklist" | ||
32 | blacklist = File.read(config["blacklist"]).split("\n") | ||
33 | all_achieves.reject! { |l| blacklist.include? l } | ||
34 | end | ||
35 | |||
36 | File.open("achieves.txt", "w") do |f| | ||
37 | f << all_achieves.join("\n") | ||
38 | end | ||