1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
require 'json'
require 'open-uri'
require 'yaml'
require 'rubygems'
require 'bundler/setup'
Bundler.require :default
MOON_COLORS = [
:blue,
:brown,
:cyan,
:green,
:orange,
:pink,
:purple,
:red,
:star,
:white,
:yellow
]
@config = YAML.load(open(ARGV[0]))
db_existed = File.exists?(@config["database"])
db = Sequel.connect("sqlite://" + @config["database"])
if ARGV[1] == "init"
if db_existed
raise "Datafile already exists"
end
schema = File.read("schema.sql")
db.run schema
puts "Initialized datafile"
exit
end
class Profile < Sequel::Model
many_to_many :achievements, join_table: :dids
end
class Game < Sequel::Model
one_to_many :achievements
one_to_many :images
end
class Achievement < Sequel::Model
many_to_one :game
many_to_many :profiles, join_table: :dids
end
class Image < Sequel::Model
many_to_one :game
end
class Did < Sequel::Model
many_to_one :profile
many_to_one :achievement
end
def scrape_profile(profile, full)
if full
url = "https://steamcommunity.com/#{profile.profile_path}/games/?tab=all"
else
url = "https://steamcommunity.com/#{profile.profile_path}/games/"
end
page = Nokogiri::HTML(open(url))
script = page.css(".responsive_page_template_content script").text[18..-1]
data = JSON.parse(script[0..script.index(";\r\n\t\t")-1])
ids = data.map { |d| d["appid"] }
index = 0
ids.each do |id|
index += 1
puts "#{profile.profile_path} - #{index}/#{ids.count}"
achsp = Nokogiri::HTML(
open("https://steamcommunity.com/#{profile.profile_path}/stats/#{id}/"))
achsp.css(".achieveTxt").each do |node|
unless node.css(".achieveUnlockTime").empty?
if Game.where(steam_appid: id).count > 0
game = Game.where(steam_appid: id).first
else
moon_index = Random.rand(MOON_COLORS.size)
game = Game.new(steam_appid: id, color: MOON_COLORS[moon_index])
game.save
storepage = Nokogiri::HTML(
open("http://store.steampowered.com/app/#{id}"))
img_id = 0
storepage.css(".highlight_screenshot_link").each do |node|
begin
imagepage = open(node["href"]).read
img_id += 1
img_filename = "#{id}-#{img_id}.jpg"
img_filepath = File.join(@config["images"], img_filename)
img_file = File.open(img_filepath, "w")
img_file.write(imagepage)
img_file.close
image = Image.new(game: game, filename: img_filename)
image.save
rescue OpenURI::HTTPError
puts "Error downloading an image"
end
sleep 2
end
end
title = node.at_css("h3").text
if game.achievements_dataset.where(title: title).count > 0
achievement = game.achievements_dataset.where(title: title).first
else
achievement = Achievement.new(game: game, title: title)
achievement.save
end
unless Did.where(profile: profile, achievement: achievement).count > 0
begin
unlock = DateTime.strptime(
node.css(".achieveUnlockTime").text.lstrip[9..-1],
"%b %d, %Y @ %l:%M%P")
rescue ArgumentError
unlock = DateTime.strptime(
node.css(".achieveUnlockTime").text.lstrip[9..-1],
"%b %d @ %l:%M%P")
end
join = Did.new(
profile: profile,
achievement: achievement,
achieved_at: unlock)
join.save
end
end
end
end
end
if ARGV[1] == "add"
userpath = ARGV[2]
if Profile.where(profile_path: userpath).count > 0
raise "Profile " + userpath + " already exists"
end
profile = Profile.new(profile_path: userpath)
profile.save
scrape_profile profile, true
elsif ARGV[1] == "update"
if ARGV.size == 3
scrape_profile Profile.where(profile_path: ARGV[2]).first, false
else
Profile.all.each do |profile|
scrape_profile profile, false
end
end
elsif ARGV[1] == "full"
if ARGV.size == 3
scrape_profile Profile.where(profile_path: ARGV[2]).first, true
else
Profile.all.each do |profile|
scrape_profile profile, true
end
end
elsif ARGV[1] == "recolor"
Game.all.each do |game|
moon_index = Random.rand(MOON_COLORS.size)
game.color = MOON_COLORS[moon_index]
game.save
end
end
|
