ruby 将所有要点同步到本地磁盘

Posted 2021-05-15

#!/usr/bin/env ruby

require 'excon'
require 'json'


module EnumerableEnumerator

  def self.included(base)
    base.extend ClassMethods

    base.eager :first, :next
    base.chainable :map, :each, :select, :find_all, :flat_map
  end

  module ClassMethods

    def chainable(*methods)
      methods.each do |method|
        send :define_method, method do |*args, &block|
          if block
            lazy_enumerator.send(method, *args, &block)
          else
            lazy_enumerator
          end
        end
      end
    end

    def eager(*methods)
      methods.each do |method|
        send :define_method, method do |*args, &block|
          lazy_enumerator.send(method, *args, &block)
        end
      end
    end

  end

  def lazy_enumerator
    enumerator.lazy
  end
end

class GistEnumerator
  include EnumerableEnumerator

  attr_reader :token, :user, :gists_dir, :page_size

  def initialize(token:, user:, gists_dir:, page_size: 100)
    @token     = token
    @user      = user
    @gists_dir = gists_dir
    @page_size = page_size
  end

  def urls
    [
      "https://api.github.com/users/#{user}/gists", # public gists
      "https://api.github.com/gists"                # private gists
    ]
  end

  def enumerator
    ::Enumerator.new do |yielder|
      urls.each do |url|
        loop do
          response = Excon.get(url,
            headers: {
              "Authorization" => "token #{token}",
              "User-Agent" => "gist-puller"
            },
            query: { per_page: page_size },
            idempotent: true,
            expects: [200]
          )

          gists = JSON.parse(response.body).map{|gist| gist["git_pull_url"]}.map do |git_pull_url|
            Gist.new(url: git_pull_url, gists_dir: gists_dir)
          end
          yielder.yield(gists)

          unless url = response.headers["Link"].scan(/<([^>]+)>; rel="([^"]+)"/).select{ |link| link[1] == "next" }.map{|link| link[0] }.first
            break
          end
        end
      end
    end

  end
end

class Gist
  attr_reader :dir, :url

  def initialize(url:, gists_dir:)
    @url = url
    @dir = File.join(gists_dir, repo)
  end

  def update
    if File.directory?(dir)
      system("git", "pull", chdir: dir)
    else
      system("git", "clone", url, dir)
    end
  end

  def repo
    url.match(/\/([^\/]+).git$/)[1]
  end
end

trap "SIGINT" do
  exit 130
end

GISTS_DIR = File.expand_path(File.dirname(File.readlink(__FILE__))  + "/..")
token = ENV['TOKEN']
user = ENV["GIST_USER"]

unless token && user
  puts <<-USAGE
Usage:
  mkdir -p gists
  cd gists
  GIST_USER='my github user' TOKEN='a github oauth token with gist scope' #{__FILE__}
USAGE

  exit 1
end

GistEnumerator.new(token: token, user: user, gists_dir: GISTS_DIR, page_size: 50).each do |gists|
  gists.inject([]){ |t, gist|
    t << Thread.new do
      gist.update
    end
  }.each(&:join)
end