Redesigned persistent data formta

This is the start of a project to add imagery to the bot's output. We began by rewriting the scraper to use a SQLite datafile instead of dumping achievement names to a text file. This allows storage of additional information about each achievement, and allows for more sophisticated scraping.

Profiles to be scraped can be added on the command line using the scraper script, instead of being specified in a config file. The scraper can conduct full or delta scrapes; in a delta scrape, only each profile's recent games are scraped, whereas they are all scraped in a full scrape.

When a game is scraped for the first time, images from the store page of that game are saved locally to be used by the bot. The bot has been altered to not use Twitter, and instead generate a pixelated image based on an image from the game of the chosen achievement. This is just for development purposes. It also crashes occasionally due to picking an achievement from a game that does not have any images saved.

Sprites of the moons from Odyssey have been included in the repository. A short message denoting their copyright is included.

1 files changed, 157 insertions, 24 deletions

@@ -1,42 +1,175 @@
 require 'json'
-require 'nokogiri'
 require 'open-uri'
 require 'yaml'
 
-config = YAML.load(open(ARGV[0]))
-usernames = config["usernames"]
+require 'rubygems'
+require 'bundler/setup'
+Bundler.require :default
 
-achieves = usernames.map do |username|
-  page = Nokogiri::HTML(open("https://steamcommunity.com/#{username}/games/?tab=all"))
+@config = YAML.load(open(ARGV[0]))
+db_existed = File.exists?(@config["database"])
+db = Sequel.connect("sqlite://" + @config["database"])
+
+if ARGV[1] == "init"
+  if db_existed
+    raise "Datafile already exists"
+  end
+
+  schema = File.read("schema.sql")
+
+  db.run schema
+
+  puts "Initialized datafile"
+
+  exit
+end
+
+class Profile < Sequel::Model
+  many_to_many :achievements, join_table: :dids
+end
+
+class Game < Sequel::Model
+  one_to_many :achievements
+  one_to_many :images
+end
+
+class Achievement < Sequel::Model
+  many_to_one :game
+  many_to_many :profiles, join_table: :dids
+end
+
+class Image < Sequel::Model
+  many_to_one :game
+end
+
+class Did < Sequel::Model
+  many_to_one :profile
+  many_to_one :achievement
+end
+
+@moonimgs = Dir.entries(@config["moon_images"]).select do |img|
+  img.end_with? ".png"
+end
+
+def scrape_profile(profile, full)
+  if full
+    url = "https://steamcommunity.com/#{profile.profile_path}/games/?tab=all"
+  else
+    url = "https://steamcommunity.com/#{profile.profile_path}/games/"
+  end
+
+  page = Nokogiri::HTML(open(url))
   script = page.css(".responsive_page_template_content script").text[18..-1]
   data = JSON.parse(script[0..script.index(";\r\n\t\t")-1])
   ids = data.map { |d| d["appid"] }
 
   index = 0
-  ids.map do |id|
+  ids.each do |id|
     index += 1
-    puts "#{username} - #{index}/#{ids.count}"
+    puts "#{profile.profile_path} - #{index}/#{ids.count}"
 
-    achsp = Nokogiri::HTML(open("https://steamcommunity.com/#{username}/stats/#{id}/"))
-    achsp.css(".achieveTxt .achieveUnlockTime + h3").map { |d| d.text }
-  end
-end.flatten
+    achsp = Nokogiri::HTML(
+      open("https://steamcommunity.com/#{profile.profile_path}/stats/#{id}/"))
 
-if File.exists?(config["achievements"])
-  already = File.read(config["achievements"]).split("\n")
-  all_achieves = achieves + already
-else
-  all_achieves = achieves
-end
+    achsp.css(".achieveTxt").each do |node|
+      unless node.css(".achieveUnlockTime").empty?
+        if Game.where(steam_appid: id).count > 0
+          game = Game.where(steam_appid: id).first
+        else
+          moon_index = Random.rand(@moonimgs.size)
+
+          game = Game.new(steam_appid: id, moon_image: @moonimgs[moon_index])
+          game.save
+
+          storepage = Nokogiri::HTML(
+            open("http://store.steampowered.com/app/#{id}"))
+
+          img_id = 0
+          storepage.css(".highlight_screenshot_link").each do |node|
+            begin
+              imagepage = open(node["href"]).read
+
+              img_id += 1
+              img_filename = "#{id}-#{img_id}.jpg"
+              img_filepath = File.join(@config["images"], img_filename)
+
+              img_file = File.open(img_filepath, "w")
+              img_file.write(imagepage)
+              img_file.close
+
+              image = Image.new(game: game, filename: img_filename)
+              image.save
+            rescue OpenURI::HTTPError
+              puts "Error downloading an image"
+            end
+
+            sleep 2
+          end
+        end
+
+        title = node.at_css("h3").text
 
-all_achieves.sort!
-all_achieves.uniq!
+        if game.achievements_dataset.where(title: title).count > 0
+          achievement = game.achievements_dataset.where(title: title).first
+        else
+          achievement = Achievement.new(game: game, title: title)
+          achievement.save
+        end
 
-if config.key? "blacklist"
-  blacklist = File.read(config["blacklist"]).split("\n")
-  all_achieves.reject! { |l| blacklist.include? l }
+        unless Did.where(profile: profile, achievement: achievement).count > 0
+          begin
+            unlock = DateTime.strptime(
+              node.css(".achieveUnlockTime").text.lstrip[9..-1],
+              "%b %d, %Y @ %l:%M%P")
+          rescue ArgumentError
+            unlock = DateTime.strptime(
+              node.css(".achieveUnlockTime").text.lstrip[9..-1],
+              "%b %d @ %l:%M%P")
+          end
+
+          join = Did.new(
+            profile: profile,
+            achievement: achievement,
+            achieved_at: unlock)
+          join.save
+        end
+      end
+    end
+  end
 end
 
-File.open(config["achievements"], "w") do |f|
-  f << all_achieves.join("\n")
+if ARGV[1] == "add"
+  userpath = ARGV[2]
+
+  if Profile.where(profile_path: userpath).count > 0
+    raise "Profile " + userpath + " already exists"
+  end
+
+  profile = Profile.new(profile_path: userpath)
+  profile.save
+
+  scrape_profile profile, true
+elsif ARGV[1] == "update"
+  if ARGV.size == 3
+    scrape_profile Profile.where(profile_path: ARGV[2]).first, false
+  else
+    Profile.all.each do |profile|
+      scrape_profile profile, false
+    end
+  end
+elsif ARGV[1] == "full"
+  if ARGV.size == 3
+    scrape_profile Profile.where(profile_path: ARGV[2]).first, true
+  else
+    Profile.all.each do |profile|
+      scrape_profile profile, true
+    end
+  end
+elsif ARGV[1] == "recolor"
+  Game.all.each do |game|
+    moon_index = Random.rand(@moonimgs.size)
+
+    game.moon_image = @moonimgs[moon_index]
+    game.save
+  end
 end