1 files changed, 157 insertions, 24 deletions

@@ -1,42 +1,175 @@
 require 'json'
-require 'nokogiri'
 require 'open-uri'
 require 'yaml'
 
-config = YAML.load(open(ARGV[0]))
-usernames = config["usernames"]
+require 'rubygems'
+require 'bundler/setup'
+Bundler.require :default
 
-achieves = usernames.map do |username|
-  page = Nokogiri::HTML(open("https://steamcommunity.com/#{username}/games/?tab=all"))
+@config = YAML.load(open(ARGV[0]))
+db_existed = File.exists?(@config["database"])
+db = Sequel.connect("sqlite://" + @config["database"])
+
+if ARGV[1] == "init"
+  if db_existed
+    raise "Datafile already exists"
+  end
+
+  schema = File.read("schema.sql")
+
+  db.run schema
+
+  puts "Initialized datafile"
+
+  exit
+end
+
+class Profile < Sequel::Model
+  many_to_many :achievements, join_table: :dids
+end
+
+class Game < Sequel::Model
+  one_to_many :achievements
+  one_to_many :images
+end
+
+class Achievement < Sequel::Model
+  many_to_one :game
+  many_to_many :profiles, join_table: :dids
+end
+
+class Image < Sequel::Model
+  many_to_one :game
+end
+
+class Did < Sequel::Model
+  many_to_one :profile
+  many_to_one :achievement
+end
+
+@moonimgs = Dir.entries(@config["moon_images"]).select do |img|
+  img.end_with? ".png"
+end
+
+def scrape_profile(profile, full)
+  if full
+    url = "https://steamcommunity.com/#{profile.profile_path}/games/?tab=all"
+  else
+    url = "https://steamcommunity.com/#{profile.profile_path}/games/"
+  end
+
+  page = Nokogiri::HTML(open(url))
   script = page.css(".responsive_page_template_content script").text[18..-1]
   data = JSON.parse(script[0..script.index(";\r\n\t\t")-1])
   ids = data.map { |d| d["appid"] }
 
   index = 0
-  ids.map do |id|
+  ids.each do |id|
     index += 1
-    puts "#{username} - #{index}/#{ids.count}"
+    puts "#{profile.profile_path} - #{index}/#{ids.count}"
 
-    achsp = Nokogiri::HTML(open("https://steamcommunity.com/#{username}/stats/#{id}/"))
-    achsp.css(".achieveTxt .achieveUnlockTime + h3").map { |d| d.text }
-  end
-end.flatten
+    achsp = Nokogiri::HTML(
+      open("https://steamcommunity.com/#{profile.profile_path}/stats/#{id}/"))
 
-if File.exists?(config["achievements"])
-  already = File.read(config["achievements"]).split("\n")
-  all_achieves = achieves + already
-else
-  all_achieves = achieves
-end
+    achsp.css(".achieveTxt").each do |node|
+      unless node.css(".achieveUnlockTime").empty?
+        if Game.where(steam_appid: id).count > 0
+          game = Game.where(steam_appid: id).first
+        else
+          moon_index = Random.rand(@moonimgs.size)
+
+          game = Game.new(steam_appid: id, moon_image: @moonimgs[moon_index])
+          game.save
+
+          storepage = Nokogiri::HTML(
+            open("http://store.steampowered.com/app/#{id}"))
+
+          img_id = 0
+          storepage.css(".highlight_screenshot_link").each do |node|
+            begin
+              imagepage = open(node["href"]).read
+
+              img_id += 1
+              img_filename = "#{id}-#{img_id}.jpg"
+              img_filepath = File.join(@config["images"], img_filename)
+
+              img_file = File.open(img_filepath, "w")
+              img_file.write(imagepage)
+              img_file.close
+
+              image = Image.new(game: game, filename: img_filename)
+              image.save
+            rescue OpenURI::HTTPError
+              puts "Error downloading an image"
+            end
+
+            sleep 2
+          end
+        end
+
+        title = node.at_css("h3").text
 
-all_achieves.sort!
-all_achieves.uniq!
+        if game.achievements_dataset.where(title: title).count > 0
+          achievement = game.achievements_dataset.where(title: title).first
+        else
+          achievement = Achievement.new(game: game, title: title)
+          achievement.save
+        end
 
-if config.key? "blacklist"
-  blacklist = File.read(config["blacklist"]).split("\n")
-  all_achieves.reject! { |l| blacklist.include? l }
+        unless Did.where(profile: profile, achievement: achievement).count > 0
+          begin
+            unlock = DateTime.strptime(
+              node.css(".achieveUnlockTime").text.lstrip[9..-1],
+              "%b %d, %Y @ %l:%M%P")
+          rescue ArgumentError
+            unlock = DateTime.strptime(
+              node.css(".achieveUnlockTime").text.lstrip[9..-1],
+              "%b %d @ %l:%M%P")
+          end
+
+          join = Did.new(
+            profile: profile,
+            achievement: achievement,
+            achieved_at: unlock)
+          join.save
+        end
+      end
+    end
+  end
 end
 
-File.open(config["achievements"], "w") do |f|
-  f << all_achieves.join("\n")
+if ARGV[1] == "add"
+  userpath = ARGV[2]
+
+  if Profile.where(profile_path: userpath).count > 0
+    raise "Profile " + userpath + " already exists"
+  end
+
+  profile = Profile.new(profile_path: userpath)
+  profile.save
+
+  scrape_profile profile, true
+elsif ARGV[1] == "update"
+  if ARGV.size == 3
+    scrape_profile Profile.where(profile_path: ARGV[2]).first, false
+  else
+    Profile.all.each do |profile|
+      scrape_profile profile, false
+    end
+  end
+elsif ARGV[1] == "full"
+  if ARGV.size == 3
+    scrape_profile Profile.where(profile_path: ARGV[2]).first, true
+  else
+    Profile.all.each do |profile|
+      scrape_profile profile, true
+    end
+  end
+elsif ARGV[1] == "recolor"
+  Game.all.each do |game|
+    moon_index = Random.rand(@moonimgs.size)
+
+    game.moon_image = @moonimgs[moon_index]
+    game.save
+  end
 end