gitmdf+

#!/usr/bin/env ruby

require 'fileutils'
require 'json'
require 'logger'
require 'sinatra/base'
require 'yaml'
require 'net/http'
require 'uri'
require 'shellwords'
require 'ipaddr'
require_relative 'server_ip_ranges'
require_relative 'sliding_window'


class GitNotifier
  STATE_FILE = '.git-notifier.dat'

  private

  MAPPINGS = {
    'from' => 'sender',
    'to' => 'mailinglist',
    'subject' => 'emailprefix',
    'uri' => 'repouri'
  }

  public

  def self.run(path, opts)
    args = Hash[opts.map { |k, v| [MAPPINGS[k] || k, v] }]
    if opts['github'] == true
        args.delete('github');
        success = pull_execute(path, args)
    else
        args.delete('github');
        success = execute(path, args)
    end
    $logger.error('git-notifier failed') unless success
    success
  end

  private

  def self.makemail(args = [])
      $file_logger.debug("#{args}")
      no_content_args = {}.merge(args)
      no_content_args.delete('content')
      $logger.debug("#{no_content_args}")

      mail_content = "Subject: #{args['emailprefix']} #{args['message'].lines.first.rstrip!}\n"\
                   "From: #{args['sender']}\n"\
                   "To: #{args['mailinglist'].join(",")}\n\n"\
                   "Repository: #{args['repouri']}\n"\
                   "On branch: #{args['branch']}\n"\
                   "Link: #{args['link']}\n"\
                   "\n>---------------------------------------------------------------\n\n"\
                   "Commit: #{args['commit_hash']}\n"\
                   "Author: #{args['author']}\n"\
                   "Date: #{args['date']}\n\n"\
                   "#{args['message']}\n"\
                   "\n>---------------------------------------------------------------\n\n"\
                   "#{args['content']}"

      return mail_content
  end

  def self.execute(path, args = [])
    mail_content = makemail(args).dump[1...-1]
    success = true
    begin
      success = system("echo #{Shellwords.escape(mail_content)} | "\
                       "/usr/sbin/sendmail #{args['mailinglist'].join(",")}")
      raise "sendmail failed in #{path} with args: #{args}" unless success
    rescue Exception => e
      $logger.error(e)
    end
    success
  end

  def self.pull_execute(path, args = [])
    args = args.map do |k, v|
      v = v * ',' if k == 'mailinglist'
      next unless v
      ["--#{k}"] + (!!v == v ? [] : ["#{v}"]) # Ignore non-boolean values.
    end
    current = Dir.pwd()
    success = true
    Dir.chdir(path)
    begin
      $logger.debug('> git fetch origin +refs/heads/*:refs/heads/*')
      success = system('git', 'fetch', 'origin', '+refs/heads/*:refs/heads/*')
      raise "git fetch failed in #{path}" unless success
      args = args.flatten.delete_if { |x| x.nil? }
      $logger.debug("> git-notifier #{args}")

      # TODO: call sendmail directly instead of git-notifier
      success = system('git-notifier', *args)
      raise "git-notifier failed in #{path} with args: #{args}" unless success
    rescue Exception => e
      $logger.error(e)
    end
    Dir.chdir(current)
    success
  end
end

class GitMdf
  def initialize(config)
    @notifier = config['notifier']
    @github = config['github']
    @github_recent_commits = RecentCommits.new(config['gitmdf']['commit_record_timespan'])
    @bitbucket = config['bitbucket']
    @bitbucket_recent_commits = RecentCommits.new(config['gitmdf']['commit_record_timespan'])
    @silent_init = config['gitmdf']['silent_init']
    @account = config['account']

    dir = config['gitmdf']['directory']
    if dir != '.'
      $logger.info("switching into working directory #{dir}")
      Dir.mkdir(dir) unless Dir.exists?(dir)
      Dir.chdir(dir)
    end
  end

  def process_github(push)
    opts = @notifier.clone
    url = push['repository']['url']
    user = push['repository']['owner']['name']
    repo = push['repository']['name']
    opts['link'] = "#{url}/compare/#{push['before']}...#{push['after']}"
    $logger.info("received push from #{user}/#{repo} for commits "\
                 "#{push['before'][0..5]}...#{push['after'][0..5]}")

    @github.each do |entry|
      if "#{user}\/#{repo}" =~ Regexp.new(entry['id'] + '$', Regexp::MULTILINE)
        opts.merge!(entry.reject { |k, v| k == 'id' || k == 'protocol'})
        opts['uri'] ||= url

        entry['protocol'] ||= 'git'
        remote = case entry['protocol']
                 when /git/
                   "git://github.com/#{user}/#{repo}.git"
                 when /ssh/
                   "git@github.com:#{user}/#{repo}.git"
                 when /https/
                   "https://github.com/#{user}/#{repo}.git"
                 else
                   $logger.error("invalid protocol: #{entry['protocol']}")
                   next
                 end

        # https://developer.github.com/v3/auth/#basic-authentication

        dir = File.join(user, repo)
        if not Dir.exists?(dir)
          $logger.debug("> git clone --bare #{remote} #{dir}")
          if not system('git', 'clone', '--bare', remote, dir)
            $logger.error("git failed to clone repository #{user}/#{repo}")
            FileUtils.rm_rf(dir) if File.exists?(dir)
            return
          end
          # Do not keep empty user directories.
          if Dir[File.join(user, '*')].empty?
            Dir.rmdir(user)
          end
        end

        state_file = File.join(dir, GitNotifier::STATE_FILE)
        if @silent_init and not File.exists?(state_file)
          $logger.info("configuring git-notifer for silent update")
          opts['updateonly'] = true unless File.exists?(state_file)
        end

        # check duplicates
        if @github_recent_commits.has(url, push['after'])
          $logger.info("ignore duplicated request from #{url} for commit "\
                       "#{commit['hash']}")
          next
        end
        @github_recent_commits.add(url, push['after'])

        opts['github'] = true
        return GitNotifier.run(dir, opts)
      end
    end
    $logger.warn("no matching repository found for #{user}/#{repo}")
  end

  def rest_bitbucket(url)
    uri = URI.parse(url);

    http = Net::HTTP.new(uri.host, uri.port)
    http.use_ssl = true
    request = Net::HTTP::Get.new(uri.request_uri)
    request["Content-Type"] = "application/json"
    request.basic_auth(@account['bitbucket']['user'],
                       @account['bitbucket']['token'])

    response = http.request(request)
    return response
  end

  def process_bitbucket(push)
    opts = @notifier.clone
    url = push['repository']['links']['html']['href']
    user = push['repository']['owner']['username']
    if user.nil? or user.empty?
        user = push['repository']['full_name'].split(/\//).first
    end
    repo = push['repository']['name']
    push_before = push['push']['changes'][0]['old']['target']['hash']
    push_after = push['push']['changes'][0]['new']['target']['hash']
    opts['link'] = "#{url}/compare/#{push_before}...#{push_after}"
    $logger.info("received push from #{user}/#{repo} for commits "\
                 "#{push_before[0..5]}...#{push_after[0..5]}")

    @bitbucket.each do |entry|
      if "#{user}\/#{repo}" =~ Regexp.new(entry['id'])
        opts.merge!(entry.reject { |k, v| k == 'id' || k == 'protocol'})

        push['push']['changes'][0]['commits'].each do |commit|
          entry['protocol'] ||= 'git'
          opts['uri'] ||= url
          opts['branch'] ||= push['push']['changes'][0]['new']['name']
          opts['author'] = commit['author']['raw']
          opts['commit_hash'] = commit['hash']
          opts['date'] = commit['date']
          opts['message'] = commit['message']

          dir = File.join(user, repo)
          # merge node is not supported correctly yet
          url = "https://api.bitbucket.org/2.0/repositories/#{dir}/diff/"\
                "#{commit['hash']}..#{commit['parents'][0]['hash']}"

          for tries in [1..3]
            response = rest_bitbucket(url);
            $logger.debug("> REST API returns #{response.code} for commit "\
                          "#{commit['hash']}")

            if response.code == '200'
              opts['content'] = response.body.gsub(/(?<=GIT\sbinary\spatch).*(?=\n\n)/m, '')
              if opts['content'] != response.body
                  opts['content'].concat("Commit contains binary; content may be incomplete\n")
              end
              break
            else
              opts['content'] = "Something was committed but I failed to pull "\
                                "the info for error #{response.code} :(\n"
            end
          end

          # check duplicates
          if @bitbucket_recent_commits.has(url, commit['hash'])
            $logger.info("ignore duplicated request from #{url} for commit "\
                         "#{commit['hash']}")
            next
          end
          @bitbucket_recent_commits.add(url, commit['hash'])

          opts['github'] = false
          ret_code = GitNotifier.run(dir, opts)
          if ret_code != true
            return ret_code
          end
        end

        return true
      end
    end
    $logger.warn("no matching repository found for #{user}/#{repo}")
  end
end

class GitMdfServer < Sinatra::Base
  configure do
    set(:environment, :production)
    set(:bind, settings.bind)
    set(:port, settings.port)
  end

  get '/' do
    "Use #{request.url} as WebHook URL in your GitHub or BitBucket repository settings."
  end

  post '/' do
    sources = settings.allowed_sources
    if not sources.nil? and not sources.empty?
      source_permit = false
      for allowed_ip in sources
        allow = IPAddr.new(allowed_ip).include?(request.ip)
        if allow
          source_permit = true
          break
        end
      end
      if not source_permit
        $logger.info("discarding request from disallowed address #{request.ip}")
        return
      end
    end

    blacklist_domains = settings.blacklist_domains
    if not blacklist_domains.nil? and not blacklist_domains.empty?
        for bad_domain in blacklist_domains
            if request.url.include?(bad_domain)
                $logger.info("discarding request from blacklist domain #{request.url}")
                return
            end
        end
    end

    if not params[:payload]
      $logger.error('received POST request with empty payload; will try parsing request body')
    else
      json = JSON.parse(params[:payload])
      if not json
        $log.error('received invalid JSON:')
        STDERR.puts(params[:payload])
      else
        STDERR.puts(JSON.pretty_generate(json)) if settings.debug_post
        # Ideally we'd use the X-Github-Event header to distinguish a ping from
        # an ordinary push. However, the 'headers' variable in Sinatra only
        # contains Content-Type, so we introspect the JSON instead.
        if json['zen']
          $logger.debug('got ping from github')
        else
          settings.gitmdf.process_github(json)
        end
      end
    end

    # BitBucket WebHook API issue:
    # https://bitbucket.org/site/master/issues/11537/webhooks-payload-empty
    if not params[:payload]
        request.body.rewind
        request_payload = request.body.read
        json = JSON.parse(request_payload)
        if not json
            $log.error('received invalid JSON:')
            STDERR.puts(request_payload)
        else
            STDERR.puts(JSON.pretty_generate(json)) if settings.debug_post
            settings.gitmdf.process_bitbucket(json)
        end
    end
  end
end

def which(cmd)
  ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
    exe = "#{path}/#{cmd}"
    return exe if File.executable?(exe)
  end
  nil
end

def run(config)
  GitMdfServer.set(:gitmdf, GitMdf.new(config))
  GitMdfServer.set(:bind, config['gitmdf']['bind'])
  GitMdfServer.set(:port, config['gitmdf']['port'])
  GitMdfServer.set(:debug_post, config['gitmdf']['debug'])
  GitMdfServer.set(:allowed_sources, config['gitmdf']['allowed_sources'])
  GitMdfServer.set(:blacklist_domains, config['gitmdf']['blacklist_domains'])

  if not config['gitmdf']['ssl']['enable']
    Sinatra.new(GitMdfServer).run!
  else
    require 'webrick/https'
    require 'openssl'

    cert = File.open(config['gitmdf']['ssl']['cert']).read
    key = File.open(config['gitmdf']['ssl']['key']).read
    webrick_options = {
      app:            GitMdfServer,
      BindAddress:    config['gitmdf']['bind'],
      Port:           config['gitmdf']['port'],
      Logger:         $logger,
      SSLEnable:      true,
      SSLCertificate: OpenSSL::X509::Certificate.new(cert),
      SSLPrivateKey:  OpenSSL::PKey::RSA.new(key),
      SSLCertName:    [['CN', WEBrick::Utils::getservername]]
    }

    Rack::Server.start(webrick_options)
  end
end

if __FILE__ == $0
  $logger = Logger.new(STDERR)
  $logger.formatter = proc do |severity, datetime, progname, msg|
      time = datetime.strftime('%Y-%m-%d %H:%M:%S')
      "[#{time}] #{severity}#{' ' * (5 - severity.size + 1)} | #{msg}\n"
  end

  $file_logger = Logger.new('commits.log', 10, 1024000)
  $file_logger.formatter = proc do |severity, datetime, progname, msg|
      time = datetime.strftime('%Y-%m-%d %H:%M:%S')
      "[#{time}] #{severity}#{' ' * (5 - severity.size + 1)} | #{msg}\n"
  end

  unless which('git-notifier')
    $logger.error('could not find git-notifier in $PATH')
    exit 1
  end

  if ARGV.size() != 1
    STDERR.puts "usage: #{$0} <config.yml>"
    exit 1
  end

  file = File.absolute_path(ARGV[0])
  config = YAML.load_file(file)

  # Append GitHub and BitBucket IP ranges
  github_ips = github_ip_ranges()
  bitbucket_ips = bitbucket_ip_ranges()
  $logger.info("Append GitHub allow list: #{github_ips}")
  $logger.info("Append BitBucket allow list: #{bitbucket_ips}")
  config['gitmdf']['allowed_sources'] = (config['gitmdf']['allowed_sources'] + github_ips + bitbucket_ips).uniq

  sinatra = Thread.new { run(config) }

  if config['gitmdf']['monitor'] > 0
    last_modified = Time.at(0)
    loop do
      mtime = File.mtime(file)
      if mtime > last_modified
        last_modified = mtime
        $logger.info("re-reading configuration file")
        config = YAML.load_file(file)
        GitMdfServer.set(:gitmdf, GitMdf.new(config))
        GitMdfServer.set(:debug_post, config['gitmdf']['debug'])
        GitMdfServer.set(:allowed_sources, config['gitmdf']['allowed_sources'])
        GitMdfServer.set(:blacklist_domains, config['gitmdf']['blacklist_domains'])
        break if config['gitmdf']['monitor'] == 0
      end

      break unless sinatra.alive?
      sleep(config['gitmdf']['monitor'])
    end
  end

  sinatra.join
end