diff options
| author | Kelly Rauchenberger <fefferburbia@gmail.com> | 2018-02-23 10:41:53 -0500 |
|---|---|---|
| committer | Kelly Rauchenberger <fefferburbia@gmail.com> | 2018-02-23 10:41:53 -0500 |
| commit | ef66322a27a7fb252b9cbf704a21cd5bd7e5a134 (patch) | |
| tree | fdc03c5e8d4ad95f03fe58954bb7e2d9c48be5bc | |
| parent | d1e935fe125e89ad30e534b9614abca256af3ee9 (diff) | |
| download | lunatic-ef66322a27a7fb252b9cbf704a21cd5bd7e5a134.tar.gz lunatic-ef66322a27a7fb252b9cbf704a21cd5bd7e5a134.tar.bz2 lunatic-ef66322a27a7fb252b9cbf704a21cd5bd7e5a134.zip | |
Added progress logging to scraper
| -rw-r--r-- | scrape.rb | 4 |
1 files changed, 4 insertions, 0 deletions
| diff --git a/scrape.rb b/scrape.rb index 92730a5..a28f4c5 100644 --- a/scrape.rb +++ b/scrape.rb | |||
| @@ -12,7 +12,11 @@ achieves = usernames.map do |username| | |||
| 12 | data = JSON.parse(script[0..script.index(";\r\n\t\t")-1]) | 12 | data = JSON.parse(script[0..script.index(";\r\n\t\t")-1]) |
| 13 | ids = data.map { |d| d["appid"] } | 13 | ids = data.map { |d| d["appid"] } |
| 14 | 14 | ||
| 15 | index = 0 | ||
| 15 | ids.map do |id| | 16 | ids.map do |id| |
| 17 | index += 1 | ||
| 18 | puts "#{username} - #{index}/#{ids.count}" | ||
| 19 | |||
| 16 | achsp = Nokogiri::HTML(open("https://steamcommunity.com/#{username}/stats/#{id}/")) | 20 | achsp = Nokogiri::HTML(open("https://steamcommunity.com/#{username}/stats/#{id}/")) |
| 17 | achsp.css(".achieveTxt .achieveUnlockTime + h3").map { |d| d.text } | 21 | achsp.css(".achieveTxt .achieveUnlockTime + h3").map { |d| d.text } |
| 18 | end | 22 | end |