Commit de77b897 authored by Lakin Wecker's avatar Lakin Wecker

Initial public domain dump.

parents
This is free and unencumbered software released into the public domain.
Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.
In jurisdictions that recognize copyright laws, the author or authors
of this software dedicate any and all copyright interest in the
software to the public domain. We intend this dedication to be an overt act of
relinquishment in perpetuity of all present and future rights to this
software under copyright law.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
For more information, please refer to <http://unlicense.org/>
Gauge Source Dump
=================
Basic notes. We aren't liable for anything you do with this code. See license.
* It's public domain.
* It's unsupported.
* You don't have all the code necessary to run.
* None of the deployment information is included.
all: bin bin/testsearch bin/createindex libs/search.so ../gamestradamus/search.so
bin:
mkdir -p bin
clean:
rm bin/testsearch bin/createindex libs/search.so src/*.o
bin/testsearch: src/testsearch.cpp
g++ -O2 -o bin/testsearch src/testsearch.cpp -Isrc/ -l pqxx -Wall -std=gnu++0x -lboost_program_options -lboost_timer -lboost_system -lboost_chrono -lrt -Iboost_1_52_0 -Lboost_1_52_0/stage/lib
bin/createindex: src/createindex.cpp
g++ -O2 -o bin/createindex src/createindex.cpp -Isrc/ -l pqxx -Wall -std=gnu++0x -lboost_program_options -lboost_timer -lboost_system -lboost_serialization -lboost_chrono -lrt -Iboost_1_52_0 -Lboost_1_52_0/stage/lib
libs/search.so: src/python-search.cpp
mkdir -p libs
g++ -std=gnu++0x -ftemplate-depth-128 -O0 -fno-inline -Wall -g -fPIC -c -o src/search.o src/python-search.cpp -Isrc/ -Iboost_1_52_0 -I../env/include/python2.7
g++ -Lboost_1_52_0/stage/lib -o libs/search.so -Wl,-h -Wl,search.so -shared -Wl,--start-group "src/search.o" -lboost_python -lboost_serialization -lboost_system -Wl,-Bstatic -Wl,-Bdynamic -lutil -lpthread -ldl -Wl,--end-group -g
../gamestradamus/search.so: libs/search.so
cp libs/search.so ../gamestradamus/crystalball/search.so
deps:
sudo apt-get install libboost1.48-all-dev g++ libpqxx-3.1 libpqxx3-dev
//------------------------------------------------------------------------------
// A program which will create the index for a particular game.
//------------------------------------------------------------------------------
#include <gamestradamus/steam/search.hpp>
#include <gamestradamus/steam/game.hpp>
#include <gamestradamus/steam/account.hpp>
#include <gamestradamus/steam/accountgame.hpp>
#include <boost/cstdint.hpp>
#include <boost/foreach.hpp>
#include <boost/program_options.hpp>
#include <boost/timer/timer.hpp>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <fstream>
#include <iostream>
#include <strstream>
#include <string>
#include <pqxx/pqxx>
using namespace gamestradamus::steam;
using namespace std;
namespace po = boost::program_options;
//------------------------------------------------------------------------------
// TODO: this connection information should be passed in from the command line
// or a config file.
std::string db_connection_string =
"dbname=xxx user=xxx password=xxx host=";
//------------------------------------------------------------------------------
// Simple helper to get the required variables from the command line
template <class T>
T required_param(const po::variables_map &vm, const std::string name) {
if (vm.count(name)) {
return vm[name].as<T>();
} else {
cerr << name << " was not set.\n";
std::exit(1);
}
}
//------------------------------------------------------------------------------
int main(int argc, char **argv) {
//--------------------------------------------------------------------------
// Parse the program options
po::options_description desc("Allowed options");
desc.add_options()
("bucket_size", po::value<int>(), "Search bucket size")
("app_id", po::value<std::string>(), "Build the index for this Game")
("filename", po::value<std::string>(), "Output file name for index")
("hostname", po::value<std::string>(), "hostname for the database")
;
po::variables_map vm;
po::store(po::parse_command_line(argc, argv, desc), vm);
po::notify(vm);
int bucket_size = required_param<int>(vm, "bucket_size");
std::string app_id = required_param<std::string>(vm, "app_id");
std::string filename = required_param<std::string>(vm, "filename");
std::string hostname = required_param<std::string>(vm, "hostname");
std::vector<AccountGame> records;
boost::timer::cpu_timer sql_time, build_time, archive_time;
//--------------------------------------------------------------------------
// GET ALL of the games and their playtimes and build the indexes.
pqxx::connection connection(db_connection_string + hostname);
{
pqxx::work W(connection, "Get the hours played for a given app_id");
// Get the game id for this particular app
string get_game_sql = "SELECT \"Game\".id AS id FROM \"Game\" WHERE \"Game\".app_id = " + app_id;
pqxx::result game = W.exec(get_game_sql);
std::string id;
game.at(0)["id"].to(id);
// Get apps marked as equivalent
std::ostringstream o;
o << "(" << id;
string get_equivalent_games = "SELECT \"Game\".id AS id FROM \"Game\" WHERE \"Game\".canonical_game_id = " + id;
pqxx::result equivalent_games = W.exec(get_equivalent_games);
for (pqxx::result::const_iterator game = equivalent_games.begin(); game != equivalent_games.end(); ++game) {
std::string game_id;
(*game)["id"].to(game_id);
o << ", " << game_id;
}
o << ")";
std::string get_hours_played = "" \
"SELECT \"UserAccountGame\".steam_hours_played AS hours_played, " \
" \"UserAccount\".openid AS openid " \
" FROM \"UserAccount\" " \
" INNER JOIN \"UserAccountGame\" " \
" ON \"UserAccountGame\".user_account_id = \"UserAccount\".id " \
" LEFT JOIN \"UserAccountAttributes\" " \
" ON \"UserAccountAttributes\".user_account_id = \"UserAccount\".id " \
" WHERE \"UserAccountGame\".game_id IN " + o.str() + \
" AND \"UserAccountGame\".steam_hours_played > 0.0 " \
" AND \"UserAccountAttributes\".private_profile_since IS NULL " \
" ORDER BY \"UserAccountGame\".steam_hours_played ASC, \"UserAccount\".openid ASC";
sql_time.start();
pqxx::result game_players = W.exec(get_hours_played);
sql_time.stop();
build_time.start();
Game::id_t game_id;
std::istringstream i(app_id);
i >> game_id;
std::string base = "http://steamcommunity.com/openid/id/";
for (pqxx::result::const_iterator player = game_players.begin(); player != game_players.end(); ++player) {
Account::id_t account_id;
std::string openid;
AccountGame::hours_t hours_played;
(*player)["openid"].to(openid);
(*player)["hours_played"].to(hours_played);
if (openid == "") {
continue;
}
std::string steam_id_64_str = openid.substr(base.length(), std::string::npos);
std::istringstream id(steam_id_64_str);
id >> account_id;
records.push_back(AccountGame(account_id, game_id, hours_played));
}
}
ConsumerTraitBucketSearch ctbs(bucket_size, records);
build_time.stop();
// Write it out to the index
archive_time.start();
std::ofstream ofs(filename);
{
boost::archive::binary_oarchive oa(ofs);
oa << ctbs;
}
archive_time.stop();
//cout << "SQL Time: " << sql_time.format();
//cout << "Build Search Tree Time: " << build_time.format();
//cout << "Archive time: " << archive_time.format();
//cout << "Done!" << endl;
}
//------------------------------------------------------------------------------
// Define the base class consumer
//
// 1. person who buys merchandise, services
//------------------------------------------------------------------------------
#ifndef GAMESTRADAMUS_CONSUMER_HPP
#define GAMESTRADAMUS_CONSUMER_HPP
#include <boost/cstdint.hpp>
namespace gamestradamus {
class Consumer {
public:
typedef boost::uint_fast64_t id_t;
};// end class Consumer
}//end namespace gamestradamus
#endif//GAMESTRADAMUS_CONSUMER_HPP
//------------------------------------------------------------------------------
// A consumer trait is a measurable sortable trait of a consumer that we can use
// to provide recommendations.
//------------------------------------------------------------------------------
#ifndef GAMESTRADAMUS_CONSUMER_TRAIT_HPP
#define GAMESTRADAMUS_CONSUMER_TRAIT_HPP
#include <gamestradamus/consumer.hpp>
namespace gamestradamus {
template <class Trait>
class ConsumerTrait {
public:
//--------------------------------------------------------------------------
ConsumerTrait() { }
//--------------------------------------------------------------------------
virtual Trait get_trait() = 0
};// end class Account
}//end namespace gamestradamus
#endif//GAMESTRADAMUS_CONSUMER_TRAIT_HPP
//------------------------------------------------------------------------------
// A steam account is a consumer
//------------------------------------------------------------------------------
#ifndef GAMESTRADAMUS_STEAM_ACCOUNT_HPP
#define GAMESTRADAMUS_STEAM_ACCOUNT_HPP
#include <gamestradamus/consumer.hpp>
namespace gamestradamus {
namespace steam {
class Account : public Consumer {
public:
//--------------------------------------------------------------------------
// Faux constructor.
Account() { }
private:
};// end class Account
}//end namespace steam
}//end namespace gamestradamus
#endif//GAMESTRADAMUS_STEAM_ACCOUNT_HPP
//------------------------------------------------------------------------------
// Define a search engine.
//------------------------------------------------------------------------------
#ifndef GAMESTRADAMUS_STEAM_ACCOUNT_GAME_HPP
#define GAMESTRADAMUS_STEAM_ACCOUNT_GAME_HPP
#include <gamestradamus/steam/game.hpp>
#include <gamestradamus/consumer.hpp>
#include <boost/serialization/access.hpp>
#include <boost/serialization/base_object.hpp>
#include <boost/serialization/utility.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/serialization/version.hpp>
namespace gamestradamus {
namespace steam {
// TODO: Make this inherit from ConsumerTrait
class AccountGame {
public:
typedef double hours_t;
//--------------------------------------------------------------------------
AccountGame()
: account_id(0), game_id(0), hours_played(0.0)
{
}
//--------------------------------------------------------------------------
AccountGame(gamestradamus::Consumer::id_t a_id, Game::id_t g_id, hours_t h_p)
: account_id(a_id), game_id(g_id), hours_played(h_p)
{
}
//--------------------------------------------------------------------------
// the account we're associated with
gamestradamus::Consumer::id_t account_id;
//--------------------------------------------------------------------------
// The Game we're associated with
Game::id_t game_id;
//--------------------------------------------------------------------------
// The hours played
hours_t hours_played;
// This below isn't technically needed yet, but eventually we'll probably
// have private variables in the future.
friend class boost::serialization::access;
template<class Archive>
void serialize(Archive & ar, const unsigned int version)
{
ar & account_id;
ar & game_id;
ar & hours_played;
}
};// end class AccountGame
// BOOST_CLASS_VERSION(AccountGame, 1)
}//end namespace steam
}//end namespace gamestradamus
#endif//GAMESTRADAMUS_STEAM_ACCOUNT_GAME_HPP
//------------------------------------------------------------------------------
// Define a search engine.
//------------------------------------------------------------------------------
#ifndef GAMESTRADAMUS_STEAM_GAME_HPP
#define GAMESTRADAMUS_STEAM_GAME_HPP
#include <boost/cstdint.hpp>
#include <boost/tuple/tuple.hpp>
using namespace boost;
namespace gamestradamus {
namespace steam {
class Game {
public:
typedef uint_fast32_t id_t;
//--------------------------------------------------------------------------
// Faux constructor.
Game() { }
private:
};// end class Game
}//end namespace steam
}//end namespace gamestradamus
#endif//GAMESTRADAMUS_STEAM_GAME_HPP
This diff is collapsed.
#include <gamestradamus/steam/game.hpp>
#include <gamestradamus/steam/search.hpp>
#include <fstream>
#include <iostream>
#include <map>
#include <boost/range/adaptors.hpp>
// include headers that implement a archive in simple binary format
#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/python.hpp>
using namespace boost::python;
using namespace gamestradamus::steam;
//------------------------------------------------------------------------------
class LocalSearch
{
public:
//--------------------------------------------------------------------------
LocalSearch(){}
//--------------------------------------------------------------------------
~LocalSearch() {
BOOST_FOREACH(ConsumerTraitBucketSearch *search, my_games | boost::adaptors::map_values) {
delete search;
}
}
//--------------------------------------------------------------------------
// Load an index file into the the search
void add_index(Game::id_t id, std::string filename) {
ConsumerTraitBucketSearch *ctbs = new ConsumerTraitBucketSearch();
{
// create and open an archive for input
std::ifstream ifs(filename.c_str());
boost::archive::binary_iarchive ia(ifs);
// read class state from archive
ia >> *ctbs;
// archive and stream closed when destructors are called
}
my_games[id] = ctbs;
}
//--------------------------------------------------------------------------
object consumers_similar_to(Game::id_t id, double hours_played, int number_of_results) {
list retval;
// TODO: there is probably a more efficient way to do this
// by not searching for id twice - but this should fix a segfault
if (my_games.find(id) == my_games.end()) {
return retval;
}
ConsumerTraitBucketSearch *ctbs = my_games[id];
match_heap_t sorted_matches = ctbs->consumers_similar_to(hours_played, number_of_results);
while (sorted_matches.size()) {
dict python_match;
Match match = sorted_matches.top();
sorted_matches.pop();
python_match["steam_id_64"] = match.account_id;
python_match["hours_played"] = match.hours_played;
retval.append(python_match);
}
return retval;
}
private:
std::map<int, ConsumerTraitBucketSearch *> my_games;
};
BOOST_PYTHON_MODULE(search)
{
class_<LocalSearch>("Search")
.def("add_index", &LocalSearch::add_index)
.def("consumers_similar_to", &LocalSearch::consumers_similar_to)
;
}
//------------------------------------------------------------------------------
// Define a python search module.
//------------------------------------------------------------------------------
#include <gamestradamus/steam/search.hpp>
#include <boost/python.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
using namespace boost::python;
#include <string>
namespace gamestradamus {
namespace python {
//------------------------------------------------------------------------------
class Search {
public:
//--------------------------------------------------------------------------
// Empty constructor
Search() {}
//--------------------------------------------------------------------------
// Empty destructor
virtual ~Search() {}
//--------------------------------------------------------------------------
void add_index(std::string filename) {
// TODO: make this an auto_ptr
ConsumerTraitBucketSearch *ctbs = new ConsumerTraitBucketSearch();
std::ifstream ifs(filename);
{
boost::archive::text_iarchive ia(ofs);
ia << ctbs;
}
my_search.add_game(ctbs);
}
private:
// Store a reference to the all game search.
AllGameSearch my_search;
}; // End class Search
}//end namespace python
}//end namespace gamestradamus
BOOST_PYTHON_MODULE(search)
{
class_<Search>("Search")
.def("add_index", &Search::add_index)
;
}
This diff is collapsed.
from gamestradamus import tools
"""Create a set of indexes locally that can be consumed by the search programs.
"""
import os
import sys
import hydro
from optparse import OptionParser
from gamestradamus import cli, units
if __name__ == "__main__":
parser = OptionParser()
parser.add_option("-d", "--directory", dest="directory",
help="Write all indexes to this DIRECTORY", metavar="DIRECTORY")
parser.add_option("-i", action="append", dest="app_ids")
parser.add_option("-c", "--config", dest="config",
help="Use the following configuration file for connection information.")
(options, args) = parser.parse_args()
app_ids = options.app_ids
if app_ids is None:
store = cli.storage_manager(config=options.config)
store.map_all(conflicts='error')
sandbox = store.new_sandbox()
app_ids = []
for g in sandbox.xrecall(units.Game, lambda g: g.is_dlc == False and g.canonical_game_id is None):
app_ids.append(int(g.app_id))
app_ids = sorted(app_ids)
db_conf = cli.get_database_configuration(config=options.config)
for id in app_ids:
print "Creating index for game %s" % (id,)
print "./cpp/bin/createindex --bucket_size=1000 --app_id=%s --filename=%s --hostname=%s" % (
id,
"%s/game_%s.crystalball" % (options.directory, id,),
db_conf['host'],
)
val = os.system("./cpp/bin/createindex --bucket_size=1000 --app_id=%s --filename=%s --hostname=%s" % (
id,
"%s/game_%s.crystalball" % (options.directory, id,),
db_conf['host'],
))
if val != 0:
sys.exit()
# -*- coding: utf-8 -*-
import sys
import string
import datetime
from geniusql import logic
from gamestradamus import cli, units, timeutils
from gamestradamus.taskqueue import tasks
from gamestradamus.bin.find_equivalent_games import find_duplicates, clean_name
#-------------------------------------------------------------------------------
if __name__ == "__main__":
"""Run this once after deploying the StatSet change to ensure they are mapped.
This will go through and look at the canonical_game_id set for all games and
update them to be a part of the same stat set.
"""
store = cli.storage_manager(conflicts='repair')
sandbox = store.new_sandbox()
games = sandbox.recall(units.Game, lambda g: g.stat_set_id is not None)
for game in games:
print "UPDATE \"Game\" SET stat_set_id=%s WHERE id=%s" % (game.stat_set_id, game.id)
store.shutdown()
# -*- coding: utf-8 -*-
import sys
import string
import datetime
from geniusql import logic
from gamestradamus import cli, units, timeutils
from gamestradamus.taskqueue import tasks
from gamestradamus.bin.find_equivalent_games import find_duplicates, clean_name
#-------------------------------------------------------------------------------
if __name__ == "__main__":
"""Run this once after deploying the StatSet change to ensure they are mapped.
This will go through and look at the canonical_game_id set for all games and