#!/usr/bin/perl -w

use strict;
use File::Basename;
use POSIX qw(strftime);

# 124.83.59.1 - dmo=10.2.81.4.1228461581268096 [05/Dec/2008:00:00:00 -0800] "GET /?product=firefox-3.0.4-complete&os=win&lang=en-US HTTP/1.1" 302 0 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1"

my $dirname = dirname($0) . '/referrer_out/';

my @locales = ('en-US', 'pt-BR', 'en-GB', 'es-ES', 'pl', 'ru');

sub get_host_from_url
{
	if($_[0] =~ m|https?://(.+?)/|) {
		return $1;
	}

	return "-";
}


my $log_re = qr|\S+ \S+ .+ \[.+-0800\] "GET (?=\S*product=firefox-2.0.0.\d+[=&\s])(?=\S*lang=(.+?)[=&\s])\S+ HTTP/..." 302 \S+ "(.+?)"|i;

my %referrers = ();

for my $loc (@locales) {
	$referrers{$loc} = {};
}

while(<STDIN>) {
	
	if($_ =~ $log_re) {
		if( exists $referrers{$1} ) {
			$referrers{$1}{$2} = 0 unless exists $referrers{$1}{$2};
			$referrers{$1}{$2}++;
		}
	}
}

while( (my $locale, my $refs) = each %referrers) {
	my %referrers_host = ();

	for my $ref_key ( keys %{$refs} ) {
		if ( exists $referrers_host{get_host_from_url($ref_key)} ) { 
			$referrers_host{get_host_from_url($ref_key)} += $referrers{$locale}{$ref_key}
		} else {
			$referrers_host{get_host_from_url($ref_key)} = $referrers{$locale}{$ref_key}
		}
	}

	my @referrers_host_tmp = sort { $referrers_host{$b} <=> $referrers_host{$a} } keys %referrers_host;
	my @referrers_tmp = sort { $referrers{$locale}{$b} <=> $referrers{$locale}{$a} } keys %{$refs};

	splice(@referrers_host_tmp, 100) unless scalar @referrers_host_tmp < 100;
	splice(@referrers_tmp, 100) unless scalar @referrers_tmp < 100;

	open(OUT_FILE, '>' . $dirname . "top_100_full_$locale.csv");
	print OUT_FILE "count,locale,referrer\n";
	for my $ref ( @referrers_tmp ) {
		print OUT_FILE $referrers{$locale}{$ref} . ",$locale,$ref\n";
	}

	open(OUT_FILE, '>' . $dirname . "top_100_host_$locale.csv");
	print OUT_FILE "count,locale,referrer\n";
	for my $ref ( @referrers_host_tmp ) {
		print OUT_FILE $referrers_host{$ref} . ",$locale,$ref\n";
	}
}
