Rakefile

require 'open3'

SERVICE='web'  # name must agree with services entry in docker-compose.yaml

# N.B. this file must be invoked from its containing directory.
# It assumes that it will be run from the top of the Whimsy code tree

def mkdir_p?(path)
  mkdir_p path unless Dir.exist? path
end

# Run system and abort if it fails
def system!(*args)
  system(*args) or raise 'system!() failed!'
end

# update gems and restart applications as needed
task :update, [:command] do |_task, args|
  # determine last update time of library sources
  lib_update = Dir['lib/**/*'].map {|n| File.mtime n rescue Time.at(0)}.max

  # restart passenger applications that have changed since the last update
  # If a Gem is later updated below, any passenger app is restarted again.
  # Most of the time, no Gems are installed, so this deploys changes quicker
  Dir['**/config.ru'].each do |rackapp|
    Dir.chdir File.dirname(rackapp) do
      old_baseline = File.mtime('tmp/restart.txt') rescue Time.at(0)
      last_update = Dir['**/*'].map {|n| File.mtime n rescue Time.at(0)}.max
      if [lib_update, last_update].max > old_baseline and Dir.exist? 'tmp'
        FileUtils.touch 'tmp/.restart.txt'
        FileUtils.chmod 0777, 'tmp/.restart.txt'
        FileUtils.mv 'tmp/.restart.txt', 'tmp/restart.txt'
      end
    end
  end

  # locate system ruby
  sysruby = File.realpath(`which ruby`.chomp)
  # N.B. The %s is used to insert related commands such as 'bundle' later on
  sysruby = File.join(File.dirname(sysruby), "%s#{sysruby[/ruby([.\d]*)$/, 1]}")

  # locate passenger ruby
  conf = Dir['/etc/apache2/*/passenger.conf'].first
  ruby = File.read(conf)[/PassengerRuby "?(.*?)"?$/, 1] if conf
  if ruby
    # create the base format string
    passruby = File.join(File.dirname(ruby), "%s#{ruby[/ruby([.\d]*)$/, 1]}")
  else
    passruby = sysruby
  end

  require 'bundler'
  unless Bundler.bundle_path.writable?
    # collect up all gems and install them so the sudo password is only
    # asked for once
    gemlines = Dir['**/Gemfile'].
      map {|file| File.read file}.join.scan(/^\s*gem\s.*/)

    if File.exist? 'asf.gemspec'
      gemlines +=
        File.read('asf.gemspec').scan(/add_dependency\((.*)\)/).
        map {|(line)| "gem #{line}"}
    end

    gems = gemlines.map {|line| [line[/['"](.*?)['"]/, 1], line.strip]}.to_h
    gems['whimsy-asf'].sub!(/,.*/, ", path: #{Dir.pwd.inspect}")

    # Also need to define version for wunderbar as per the asf.gemspec file
    require 'tmpdir'
    Dir.mktmpdir do |dir|
      Dir.chdir dir do
        contents = [
          "source 'https://rubygems.org'",
          'ldapversion = nil', # Needed for initial gem setup
          gems.values
        ].join("\n")
        File.write 'Gemfile', contents
        $stderr.puts '* Preloading gems...'
        system!('bundle', 'install')
        $stderr.puts '* ... done'
      end
    end
  end

  # update gems
  $stderr.puts '* Update Gems' # use stderr so output appears in syslog
  Dir['**/Gemfile'].each do |gemfile|
    Dir.chdir File.dirname(gemfile) do
      ruby = File.read('Gemfile')[/^ruby ['"](.*?)['"]/, 1]
      ruby = `which ruby#{ruby}`.chomp if ruby
      if ruby and not ruby.empty?
        bundler = File.join(File.dirname(ruby), "bundle#{ruby[/ruby([.\d]*)$/, 1]}")
      else
        bundler = (File.exist?('config.ru') ? passruby : sysruby) % 'bundle'
      end

      locktime = File.mtime('Gemfile.lock') rescue Time.at(0)

      bundler = 'bundle' unless File.exist?(bundler)
      $stderr.puts "* Processing #{gemfile}"
      system!(bundler, args.command || 'update')

      # if new gems were installed and this directory contains a passenger
      #  application, restart it
      if (File.mtime('Gemfile.lock') rescue Time.at(0)) != locktime
        $stderr.puts '* Gemfile.lock was updated'
        if File.exist?('tmp/restart.txt')
          FileUtils.touch 'tmp/.restart.txt'
          FileUtils.chmod 0o777, 'tmp/.restart.txt'
          FileUtils.mv 'tmp/.restart.txt', 'tmp/restart.txt'
        end
      end
    end
  end

  # rebuild API documentation
  Rake::Task['rdoc'].invoke
end

# pristine version of update
task :pristine do
  Rake::Task[:update].invoke('pristine')
end

# This requires Gems such as Wunderbar to have been set up
task :config do
  $LOAD_PATH.unshift 'lib'
  require 'wunderbar'
  require 'whimsy/asf/config'
  require 'whimsy/asf/git'
  require 'whimsy/asf/svn'
  require 'whimsy/lockfile'
end

namespace :svn do
  task :update, [:arg1] => :config do |task, args|
    arg1 = args.arg1 || '' # If defined, it is either the name of a checkout to update or 'skip'
    options = [arg1, args.extras].flatten # capture all options
    # Include all
    svnrepos = ASF::SVN.repo_entries(true) || {}

    # must be outside loop
    PREFIX = '#!:' # must agree with monitors/svn.rb

    # checkout/update svn repositories
    svn = ASF::Config.get(:svn)
    svn = Array(svn).find {|path| String === path and path.end_with? '/*'}
    if svn.instance_of? String and svn.end_with? '/*'
      mkdir_p? File.dirname(svn)
      Dir.chdir File.dirname(svn) do
        svnrepos.each do |name, description|
          # skip the update unless it matches the parameter (if any) provided
          # 'skip' is special and means update all list files
          # The empty string means no options provided
          next unless ['skip', ''].include?(arg1)  || options.include?(name)
          puts
          puts File.join(Dir.pwd, name)
          if description['list']
            puts "#{PREFIX} Updating listing file"
            old,new = ASF::SVN.updatelisting(name,nil,nil,description['dates'])
            if old == new
              puts "List is at revision #{old}."
            elsif old.nil?
              puts new
            else
              puts "List updated from #{old} to revision #{new}."
            end
          end
          svnpath = ASF::SVN.svnurl(name)
          depth = description['depth'] || 'infinity'
          noCheckout = %w(delete skip).include? depth
          if Dir.exist? name
            if noCheckout
              puts "#{PREFIX} Removing #{name} as it is not intended for checkout"
              FileUtils.rm_rf name # this will remove symlink only (on macOS at least)
            else
              curpath = ASF::SVN.getInfoItem(name,'url')
              if curpath != svnpath
                puts "Removing #{name} to correct URL: #{curpath} => #{svnpath}"
                FileUtils.rm_rf name  # this will remove symlink only (on macOS at least)
              end
            end
          end

          next if arg1 == 'skip'
          if noCheckout
            puts 'Skipping' if depth == 'skip' # Must agree with monitors/svn.rb
            next
          end

          files = description['files']
          if Dir.exist? name
            isSymlink = File.symlink?(name) # we don't want to change such checkouts
            Dir.chdir(name) {
             # ensure single-threaded SVN updates
             LockFile.lockfile(Dir.pwd, nil, File::LOCK_EX) do # ignore the return parameter
              system!('svn', 'cleanup')
              unless isSymlink # Don't change depth for symlinks
                curdepth = ASF::SVN.getInfoAsHash('.')['Depth'] || 'infinity' # not available as separate item
                if curdepth != depth
                  puts "#{PREFIX} update depth from '#{curdepth}' to '#{depth}'"
                  system!('svn', 'update', '--set-depth', depth)
                end
              end
              outerr = nil
              # svn update can fail sometimes, so we retry
              2.times do |i|
                if i > 0
                  # log the failure - prefix tells monitor to ignore it
                  puts "#{PREFIX} failed!"
                  outerr.split("\n").each do |l|
                    puts "#{PREFIX} #{l}"
                  end
                  n = 10
                  puts "#{PREFIX} will retry in #{n} seconds"
                  sleep n
                end
                begin
                  r, w = IO.pipe
                  # Note: list the files to update to cater for later additions
                  # Also update '.' so parent directory shows last changed revision for status/svn page
                  svncmd = %w(svn update .)
                  # '.' is redundant if files not present, but it simplifies logic
                  if files
                    svncmd += files
                  end
                  puts "#{PREFIX} #{svncmd.join(' ')}"
                  _pid = Process.spawn(*svncmd, out: w, err: [:child, :out])
                  w.close

                  _pid, status = Process.wait2
                  outerr = r.read
                  r.close

                  if status.success?
                    break
                  end
                rescue StandardError => e
                  outerr = e.inspect
                  break
                end
              end

              puts outerr # show what happened last
             end # lockfile
            } # chdir
          else # directory does not exist
            # Don't bother locking here -- it should be very rarely needed
            system!('svn', 'checkout', "--depth=#{depth}", svnpath, name)
            if files
              system!('svn', 'update', *files, {chdir: name})
            end
          end
          # check that explicitly required files exist
          files&.each do |file|
            path = File.join(name, file)
            puts "Missing: #{path}" unless File.exist? path
          end
        end
      end
    end
  end

  task :check => :config do
    # check if the svn repositories have been set up OK
    svnrepos = ASF::SVN.repo_entries || {}
    errors = 0
    svn = ASF::Config.get(:svn)
    if svn.instance_of? String and svn.end_with? '/*'
      Dir.chdir File.dirname(svn) do
        svnrepos.each do |name, description|
          puts
          puts File.join(Dir.pwd, name)
          if Dir.exist? name
            hash, err = ASF::SVN.getInfoAsHash(name)
            if hash
              urlact = hash['URL']
              urlexp = description['url']
              unless urlact.end_with? urlexp # urlexp is relative only
                puts "URL: #{urlact} expected to end with #{urlexp}"
                errors += 1
              end
              depthact = hash['Depth'] || 'infinity'
              depthexp = description['depth'] || 'infinity'
              unless depthact == depthexp
                puts "Depth: #{depthact} expected to be #{depthexp}"
                errors += 1
              end
            else
              puts "Error getting details for #{name}: #{err}"
              errors += 1
            end
          else
            puts "Directory not found - expecting checkout of #{ASF::SVN.svnpath!(name)}"
            errors += 1
          end
        end
      end
      puts
      if errors > 0
        puts "** Found #{errors} error(s) **"
      else
        puts '** No errors found **'
      end
    end
  end

end

namespace :git do
  task :pull => :config do
    gitrepos = ASF::Git.repo_entries() || {}

    # clone/pull git repositories
    git = ASF::Config.get(:git)
    if git.instance_of? String and git.end_with? '/*'
      mkdir_p? File.dirname(git)
      Dir.chdir File.dirname(git) do
        require 'uri'
        base = URI.parse('git://git.apache.org/')
        gitrepos.each do |name, description|
          unless description
            puts "Skipping git:pull of #{name} because no details were found"
            next
          end
          branch = description['branch']

          puts
          puts File.join(Dir.pwd, name)

          if Dir.exist? name
            Dir.chdir(name) do
              # update the location of the remote, if necessary
              remote = `git config --get remote.origin.url`.chomp
              if remote != (base + description['url']).to_s
                `git config remote.origin.url #{base + description['url']}`
              end

              # pull changes
              system!('git', 'checkout', branch) if branch
              system!('git', 'fetch', 'origin')
              system!('git', 'reset', '--hard', "origin/#{branch || 'master'}")
            end
          else
            depth = description['depth']

            # fresh checkout
            if depth
              system!('git', 'clone', '--depth', depth.to_s, (base + description['url']).to_s, name)
            else
              system!('git', 'clone', (base + description['url']).to_s, name)
            end
            system!('git', 'checkout', branch, {chdir: name}) if branch
          end
        end
      end
    end
  end
end

# update documentation
task :rdoc => 'www/docs/api/index.html'
file 'www/docs/api/index.html' => Rake::FileList['lib/whimsy/**/*.rb'] do
  # remove old files first
  FileUtils.remove_dir(File.join(File.dirname(__FILE__),'www/docs/api'), true) # ignore error if missing
  system!('rdoc', 'lib/whimsy', '--output', 'www/docs/api', '--force-output',
    '--title', 'whimsy/asf lib', {chdir: File.dirname(__FILE__)})
end

# Travis support: run the tests associated with the bundle in question
task :default do
  bg = ENV['BUNDLE_GEMFILE']
  if bg and bg != __FILE__
    Dir.chdir File.dirname(bg) do
      sh 'rake test'
    end
  end
end

# Temporary files used to propagate settings into container
LDAP_HTTPD_PATH = '../.ldap_httpd.tmp'
LDAP_WHIMSY_PATH = '../.ldap_whimsy.tmp'

# Allow use of security database on macOS
# Keychain needs to be set up with an application password
# with the Account value of the user_dn
def getpass(user_dn)
  pw = $stdin.getpass("password for #{user_dn}: ")
  return pw unless pw == '*'
  if RbConfig::CONFIG['host_os'].start_with? 'darwin'
    pw, status = Open3.capture2('security', 'find-generic-password', '-a', user_dn, '-w')
    raise "ERROR: problem running security: #{status}" unless status.success?
  else
    raise "ERROR: sorry, don't know how to get password from secure storage"
  end
  return pw.strip
end

def ldap_init
  $LOAD_PATH.unshift 'lib'
  require 'io/console' # cannot prompt from container, so need to do this upfront
  require 'whimsy/asf/config'

  whimsy_dn = ASF::Config.get(:whimsy_dn) or raise 'ERROR: Must provide whimsy_dn value in .whimsy'
  whimsy_pw = getpass(whimsy_dn)
  raise 'ERROR: Password is required' unless whimsy_pw.size > 1

  httpd_dn = ASF::Config.get(:httpd_dn)
  if httpd_dn
    httpd_pw = getpass(httpd_dn)
    raise 'ERROR: Password is required' unless httpd_pw.size > 1
  else # default to whimsy credentials
    httpd_dn = whimsy_dn
    httpd_pw = whimsy_pw
  end
  File.open(LDAP_HTTPD_PATH, 'w', 0o600) do |w|
    w.puts httpd_dn
    w.puts httpd_pw
  end
  File.open(LDAP_WHIMSY_PATH, 'w', 0o600) do |w|
    w.puts whimsy_dn
    w.puts whimsy_pw
  end
end

# Process template files replacing variable references
def filter(src, dst, ldaphosts, ldapbinddn, ldapbindpw)
  require 'erb'
  template = ERB.new(File.read(src))
  File.open(dst, 'w') do |w|
    w.write(template.result(binding))
  end
end

# Set up LDAP items in container context
def ldap_setup
  # Link to file in running container
  FileUtils.cp LDAP_WHIMSY_PATH, '/tmp/ldap.tmp'
  FileUtils.rm_f LDAP_WHIMSY_PATH # remove work file
  FileUtils.chown 'www-data', 'www-data', '/tmp/ldap.tmp'
  ln_sf '/tmp/ldap.tmp', '/srv/ldap.txt'

  ldapbinddn = ldapbindpw = nil
  File.open(LDAP_HTTPD_PATH, 'r') do |r|
    ldapbinddn = r.readline.strip
    ldapbindpw = r.readline.strip
  end
  FileUtils.rm_f LDAP_HTTPD_PATH # remove work file

  $LOAD_PATH.unshift 'lib'
  require 'whimsy/asf/config'
  hosts = ASF::Config.get(:ldap)
  raise 'ERROR: Must define :ldap in ../.whimsy' unless hosts

  ldaphosts = hosts.join(' ').gsub('ldaps://', '')

  filter('docker-config/whimsy.conf',
    '/etc/apache2/sites-enabled/000-default.conf', ldaphosts, ldapbinddn, ldapbindpw)
  filter('docker-config/25-authz_ldap_group_membership.conf',
    '/etc/apache2/conf-enabled/25-authz_ldap_group_membership.conf', ldaphosts, ldapbinddn, ldapbindpw)
  # Add the URI and BASE for use by ldapsearch from shell
  File.open("/etc/ldap/ldap.conf",'a+') do |f|
    f.puts "URI #{hosts.join(' ')}"
    f.puts "BASE dc=apache,dc=org"
  end
end

# Docker support
namespace :docker do
  task :build do
    sh "docker compose build #{SERVICE}"
  end

  task :update => :build do
    sh 'docker compose run  --entrypoint ' +
      %('bash -c "rake update"') +
      " #{SERVICE}"
  end

  task :up do
    ldap_init # create LDAP config data files
    # Start the container which then runs 'rake docker:entrypoint'
    sh 'docker compose up'
  end

  task :exec do
    sh "docker compose exec #{SERVICE} /bin/bash"
  end

  # cannot depend on :config
  # It runs in container, and needs to occur first
  task :scaffold do

    # This should already exist, but just in case
    mkdir_p? '/srv/whimsy/www/members'

    unless File.exist? '/srv/whimsy/www/members/log'
      ln_s '/var/log/apache2', '/srv/whimsy/www/members/log'
    end

    begin
      mode = File.stat('/var/log/apache2').mode
      if mode & 7 != 5
        chmod 0o755, '/var/log/apache2'
      end
      # ensure log files are readable
      sh 'chmod 0644 /var/log/apache2/*.log'
    rescue StandardError => e
      puts e.inspect
    end

    # Create other needed directories
    mkdir_p? '/srv/cache'
    mkdir_p? '/srv/mail/secretary'
    # The list-* files are pushed from the mailing list server to the live Whimsy
    # ensure there are empty files here
    mkdir_p? '/srv/subscriptions'
    Dir.chdir '/srv/subscriptions' do
      # start is done first by the server
      %w{start allows counts denys digests flags mods sendsubscribertomods subs}.each do |suffix|
        file = "list-#{suffix}"
        FileUtils.touch file unless File.exist? file
      end
    end
    # in case
    mkdir_p? '/srv/whimsy/www/docs/api'
    # there may be more

    # add support for CLI use
    unless File.exist? '/root/.bash_aliases'
      ln_s '/srv/.bash_aliases', '/root/.bash_aliases'
    end

    # Allow logs to be written to host system
    if Dir.exist? '/srv/apache2_logs'
      FileUtils.rm_rf '/var/log/apache2'
      ln_s '/srv/apache2_logs', '/var/log/apache2'
    end

    ldap_setup # set up LDAP entries in container
  end

  # This is the entrypoint in the Dockerfile so runs in the container
  task :entrypoint => [:scaffold] do
    sh 'apache2ctl -DFOREGROUND'
  end
end