diff options
| author | Kelly Rauchenberger <fefferburbia@gmail.com> | 2018-02-23 09:18:01 -0500 |
|---|---|---|
| committer | Kelly Rauchenberger <fefferburbia@gmail.com> | 2018-02-23 09:18:01 -0500 |
| commit | b77ab7cb283e64fb8b23f4892115644e84596842 (patch) | |
| tree | b5ecb460e5c10ce51df2594ba280e5d332e2cd2f /scrape.rb | |
| download | lunatic-b77ab7cb283e64fb8b23f4892115644e84596842.tar.gz lunatic-b77ab7cb283e64fb8b23f4892115644e84596842.tar.bz2 lunatic-b77ab7cb283e64fb8b23f4892115644e84596842.zip | |
Created bot
Diffstat (limited to 'scrape.rb')
| -rw-r--r-- | scrape.rb | 38 |
1 files changed, 38 insertions, 0 deletions
| diff --git a/scrape.rb b/scrape.rb new file mode 100644 index 0000000..5d62e89 --- /dev/null +++ b/scrape.rb | |||
| @@ -0,0 +1,38 @@ | |||
| 1 | require 'json' | ||
| 2 | require 'nokogiri' | ||
| 3 | require 'open-uri' | ||
| 4 | require 'yaml' | ||
| 5 | |||
| 6 | config = YAML.load(open("config.yml")) | ||
| 7 | usernames = config["usernames"] | ||
| 8 | |||
| 9 | achieves = usernames.map do |username| | ||
| 10 | page = Nokogiri::HTML(open("https://steamcommunity.com/id/#{username}/games/?tab=all")) | ||
| 11 | script = page.css(".responsive_page_template_content script").text[18..-1] | ||
| 12 | data = JSON.parse(script[0..script.index(";\r\n\t\t")-1]) | ||
| 13 | ids = data.map { |d| d["appid"] } | ||
| 14 | |||
| 15 | ids.map do |id| | ||
| 16 | achsp = Nokogiri::HTML(open("https://steamcommunity.com/id/#{username}/stats/#{id}/")) | ||
| 17 | achsp.css(".achieveTxt .achieveUnlockTime + h3").map { |d| d.text } | ||
| 18 | end | ||
| 19 | end.flatten | ||
| 20 | |||
| 21 | if File.exists?("achieves.txt") | ||
| 22 | already = File.read("achieves.txt").split("\n") | ||
| 23 | all_achieves = achieves + already | ||
| 24 | else | ||
| 25 | all_achieves = achieves | ||
| 26 | end | ||
| 27 | |||
| 28 | all_achieves.sort! | ||
| 29 | all_achieves.uniq! | ||
| 30 | |||
| 31 | if config.key? "blacklist" | ||
| 32 | blacklist = File.read(config["blacklist"]).split("\n") | ||
| 33 | all_achieves.reject! { |l| blacklist.include? l } | ||
| 34 | end | ||
| 35 | |||
| 36 | File.open("achieves.txt", "w") do |f| | ||
| 37 | f << all_achieves.join("\n") | ||
| 38 | end | ||
