From b0a54f387866450e09d41ee240d27fcd7bb110bc Mon Sep 17 00:00:00 2001 From: Todor Ivanov Date: Thu, 23 May 2024 23:26:59 +0300 Subject: [PATCH] Adding init_tier0 function (#1466) * Adding init_tier0 * Fix parsing bugs for commented lines && semicolons in values * Do not copy the manage script to WMA_CONFIG_DIR * Temporarily stop modifying agent configuration with tier0-mod-config * Fix few forgotten Tier0 related vars && reenable all steps for tier0-mod-config * Add _proxy_renew for T0 && avoid _check_mounts for T0 && Skip Add resource control for T0 && Fix variables in tier0_init * Move init_agent echo line * Fix indentation in init_tier0 * Fix indentation in init_wmagent * Rename resRecord variables * Avoid copying the run time scripts if not running from inside a Docker container * Dynamically build resource options Fix resRecord array name substitution * Add a check for docker container during initialization * Fix allways nonError return values for init_tier0 step * Avoid checking for mounts when not running from docker * Move _copy_runtime as a separate function && Fix ambiguos comments, variable names and printouts * Rename dockerBuildId to wmaBuildId && add Vir env agent start message * Follow up on review comments * Remove no longer needed export for WMAGENT_CONFIG * Follow up on review comments --- docker/pypi/wmagent/Dockerfile | 2 +- docker/pypi/wmagent/README.md | 2 +- docker/pypi/wmagent/bin/manage | 98 +++++++++-- docker/pypi/wmagent/bin/manage-common.sh | 49 +++++- docker/pypi/wmagent/init.sh | 200 ++++++++++++++--------- docker/pypi/wmagent/install.sh | 10 +- 6 files changed, 252 insertions(+), 109 deletions(-) diff --git a/docker/pypi/wmagent/Dockerfile b/docker/pypi/wmagent/Dockerfile index 00b9f762c..1b0e9bbcb 100644 --- a/docker/pypi/wmagent/Dockerfile +++ b/docker/pypi/wmagent/Dockerfile @@ -20,9 +20,9 @@ ENV WMA_INSTALL_DIR=$WMA_CURRENT_DIR/install ENV WMA_STATE_DIR=$WMA_CURRENT_DIR/state ENV WMA_CONFIG_DIR=$WMA_CURRENT_DIR/config ENV WMA_CONFIG_FILE=$WMA_CONFIG_DIR/config.py -ENV WMA_MANAGE_DIR=$WMA_CONFIG_DIR ENV WMA_LOG_DIR=$WMA_CURRENT_DIR/logs ENV WMA_DEPLOY_DIR=/usr/local +ENV WMA_MANAGE_DIR=$WMA_DEPLOY_DIR/bin ENV WMA_ENV_FILE=$WMA_DEPLOY_DIR/deploy/env.sh ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets ENV ORACLE_PATH=$WMA_DEPLOY_DIR/etc/oracle diff --git a/docker/pypi/wmagent/README.md b/docker/pypi/wmagent/README.md index 946112de8..42794e604 100644 --- a/docker/pypi/wmagent/README.md +++ b/docker/pypi/wmagent/README.md @@ -322,7 +322,7 @@ They are set at different moments along the process: i.e. at `buildtime` - when * config=$WMA_CONFIG_DIR * manage=$WMA_MANAGE_DIR/manage * RUCIO_HOME=$WMA_CONFIG_DIR - * WMA_BUILD_ID=$(cat $WMA_ROOT_DIR/.dockerBuildId) + * WMA_BUILD_ID=$(cat $WMA_ROOT_DIR/.wmaBuildId) * WMCORE_ROOT=$WMA_DEPLOY_DIR * WMAGENTPY3_ROOT=$WMA_INSTALL_DIR * WMAGENTPY3_VERSION=$WMA_TAG diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage index b56279462..e7c6547c6 100755 --- a/docker/pypi/wmagent/bin/manage +++ b/docker/pypi/wmagent/bin/manage @@ -74,7 +74,12 @@ activate_agent(){ # The very first step of activating the WMAgent upon all needed checks and # validations performed during the initialization process. # No .init* flags are checked at this stage. - cp -vf $WMA_DEPLOY_DIR/etc/WMAgentConfig.py $WMA_CONFIG_DIR/config-template.py + if [[ "$TEAMNAME" == Tier0* ]]; then + cat $WMA_DEPLOY_DIR/etc/WMAgentConfig.py $WMA_DEPLOY_DIR/etc/Tier0Config.py > $WMA_CONFIG_DIR/config-template.py + else + cp -vf $WMA_DEPLOY_DIR/etc/WMAgentConfig.py $WMA_CONFIG_DIR/config-template.py + fi + } # @@ -103,7 +108,7 @@ db_prompt(){ # Agent stuff # ############################## -# generate the agent config from the basic template +# generate the WMAgent config from the basic template init_wmagent(){ case $AGENT_FLAVOR in @@ -124,7 +129,7 @@ init_wmagent(){ local errVal=0 wmagent-mod-config $database_options \ - --input=$WMA_CONFIG_DIR/config-template.py \ + --input=$WMA_CONFIG_DIR/config-template.py \ --output=$WMA_CONFIG_DIR/config.py \ --working_dir=$WMA_INSTALL_DIR \ --couch_url=http://$COUCH_USER:$COUCH_PASS@$COUCH_HOST:$COUCH_PORT \ @@ -134,17 +139,17 @@ init_wmagent(){ --grafana_token=$GRAFANA_TOKEN \ --wmstats_url=$WMSTATS_URL \ --reqmgr2_url=$REQMGR2_URL \ - --acdc_url=$ACDC_URL \ - --dbs3_url=$DBS3_URL \ - --dbs3_reader_url=$DBS3_READER_URL \ - --dqm_url=$DQM_URL \ - --requestcouch_url=$REQUESTCOUCH_URL \ - --central_logdb_url=$CENTRAL_LOGDB_URL \ - --wmarchive_url=$WMARCHIVE_URL \ - --amq_credentials=$AMQ_CREDENTIALS \ - --rucio_account=$RUCIO_ACCOUNT \ - --rucio_host=$RUCIO_HOST \ - --rucio_auth=$RUCIO_AUTH + --acdc_url=$ACDC_URL \ + --dbs3_url=$DBS3_URL \ + --dbs3_reader_url=$DBS3_READER_URL \ + --dqm_url=$DQM_URL \ + --requestcouch_url=$REQUESTCOUCH_URL \ + --central_logdb_url=$CENTRAL_LOGDB_URL \ + --wmarchive_url=$WMARCHIVE_URL \ + --amq_credentials=$AMQ_CREDENTIALS \ + --rucio_account=$RUCIO_ACCOUNT \ + --rucio_host=$RUCIO_HOST \ + --rucio_auth=$RUCIO_AUTH let errVal+=$? wmcore-db-init --config $WMA_CONFIG_DIR/config.py --create --modules=WMCore.WMBS,WMCore.Agent.Database,WMComponent.DBS3Buffer,WMCore.BossAir,WMCore.ResourceControl; @@ -157,10 +162,71 @@ init_wmagent(){ } +############################## +# Tier0 stuff # +############################## + + +# generate the tier0 config from the basic template +init_tier0(){ + local errVal=0 + wmagent-mod-config --input=$WMA_CONFIG_DIR/config-template.py \ + --output=$WMA_CONFIG_DIR/config-agent.py \ + --coredb_url=oracle://$ORACLE_USER:$ORACLE_PASS@$ORACLE_TNS \ + --working_dir=$WMA_INSTALL_DIR \ + --couch_url=http://$COUCH_USER:$COUCH_PASS@$COUCH_HOST:$COUCH_PORT \ + --workqueue_db_name=$LOCAL_WORKQUEUE_DBNAME \ + --workload_summary_url=$WORKLOAD_SUMMARY_URL \ + --grafana_token=$GRAFANA_TOKEN \ + --wmstats_url=$WMSTATS_URL \ + --dbs3_url=$DBS3_URL \ + --dqm_url=$DQM_URL \ + --requestcouch_url=$REQUESTCOUCH_URL \ + --central_logdb_url=$CENTRAL_LOGDB_URL \ + --wmarchive_url=$WMARCHIVE_URL \ + --amq_credentials=$AMQ_CREDENTIALS \ + --rucio_account=$RUCIO_ACCOUNT \ + --rucio_host=$RUCIO_HOST \ + --rucio_auth=$RUCIO_AUTH + let errVal+=$? + + # NOTE: For the next step relates to modifying the originally generated WMAgent config file + # from $WMA_CONFIG_DIR/config-agent.py + # to $WMA_CONFIG_DIR/config.py + # It removes the components not relevant for T0 and adds the additional DB credentials + # used at Tier0Feeder. In order for it to be executed properly we need to download the + # tier0-mod-config script from https://github.com/dmwm/T0/blob/master/bin/tier0-mod-config + optargs="" + [[ -n $CONFDB_URL ]] && optargs="$optargs --confdb_url=$CONFDB_URL" + [[ -n $SMDB_URL ]] && optargs="$optargs --smdb_url=$SMDB_URL" + [[ -n $POPCONLOGDB_URL ]] && optargs="$optargs --popconlogdb_url=$POPCONLOGDB_URL" + [[ -n $T0DATASVCDB_URL ]] && optargs="$optargs --t0datasvcdb_url=$T0DATASVCDB_URL" + [[ -n $SMNOTIFYDB_URL ]] && optargs="$optargs --smnotifydb_url=$SMNOTIFYDB_URL" + + tier0-mod-config --input=$WMA_CONFIG_DIR/config-agent.py \ + --output=$WMA_CONFIG_DIR/config.py \ + $optargs + let errVal+=$? + + wmcore-db-init --config $WMA_CONFIG_DIR/config.py --create --modules=WMCore.WMBS,WMCore.Agent.Database,WMComponent.DBS3Buffer,WMCore.BossAir,WMCore.ResourceControl,T0.WMBS; + let errVal+=$? + + wmagent-couchapp-init; + let errVal+=$? + + return $errVal +} + + init_agent(){ if _init_valid $wmaInitActive ; then - echo "Initialising Agent..." - init_wmagent + if [[ "$TEAMNAME" == Tier0* ]]; then + echo "Initializing Tier0 Agent..." + init_tier0 + else + echo "Initializing WMAgent..." + init_wmagent + fi else echo "ERROR: This agent has not been activated yet. Cannot proceed." return $(false) diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh index 1790777a0..bf70820da 100644 --- a/docker/pypi/wmagent/bin/manage-common.sh +++ b/docker/pypi/wmagent/bin/manage-common.sh @@ -31,6 +31,16 @@ wmaSchemaFile=$WMA_CONFIG_DIR/.wmaSchemaFile.sql wmaDBName=wmagent #------------------------------------------------------------------------------- +_is_venv() { + # Auxiliary function to determine if we are running within a shell with active python virtual env + $(python3 -c 'import sys; isVenv = sys.prefix == sys.base_prefix;sys.exit(isVenv)') +} + +_is_docker() { + # Auxiliary function to determine if we are running within a docker env + [[ -f /.dockerenv ]] && grep -q docker /proc/1/cgroup +} + _exec_mysql() { # Auxiliary function to avoid repetitive and long calls to the mysql command # :param $1: SQL command to be executed passed a s single string @@ -252,7 +262,7 @@ _status_of_oracle(){ # Auxiliary function to check if the oracle database configured for the current agent is empty echo "$FUNCNAME:" _exec_oracle "select 1 from dual;" -# sqlplus $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS </dev/null) [[ " $mounts " =~ ^.*[[:space:]]+$mountPoint[[:space:]]+.*$ ]] && return $(true) || return $(false) @@ -98,54 +101,60 @@ basic_checks() { echo } +_copy_runtime() { + # Auxiliary function to copy all scripts needed at runtime from the WMAgent image to the host + + # Avoid copying the run time scripts if not running from inside a Docker container + _is_venv && { + echo $WMA_BUILD_ID > $wmaInitRuntime + return + } + + # checking if $WMA_DEPLOY_DIR is root path for $pythonLib: + if [[ $pythonLib =~ ^$WMA_DEPLOY_DIR ]]; then + mkdir -p $WMA_INSTALL_DIR/Docker/ + echo "$FUNCNAME: Copying content from: $pythonLib/WMCore/WMRuntime to: $WMA_INSTALL_DIR/Docker/" + cp -ra $pythonLib/WMCore/WMRuntime $WMA_INSTALL_DIR/Docker/ + echo "$FUNCNAME: Copying content from: $WMA_DEPLOY_DIR/etc/ to: $WMA_CONFIG_DIR/" + cp -ra $WMA_DEPLOY_DIR/etc/ $WMA_CONFIG_DIR/ + echo $WMA_BUILD_ID > $wmaInitRuntime + else + echo "$FUNCNAME: ERROR: \$WMA_DEPLOY_DIR: $WMA_DEPLOY_DIR is not a root path for \$pythonLib: $pythonLib" + echo "$FUNCNAME: ERROR: We cannot find the correct WMCore/WMRuntime source to copy at the current host!" + return $(false) + fi +} + deploy_to_host(){ - # This function does all the needed Docker image to Host modifications at Runtime - # DONE: Here to execute all local config and manage/copy opertaions from the image deploy area of the container to the host - # * creation of all config directories if missing at the mount point - # * reimplement init_install_dir - # * reimplement init_config_dir - # * copy/override the manage file at the host mount point with the manage file from the image deployment area - # * copy/override all config files if the agent have never been initialised - # * create/touch a .dockerInit file containing the WMA_BUILD_ID of the current docker image - # * eventually the docker container Id may be considered in the future as well (the unique hash id to be used not the contaner name) - # - # NOTE: On every step we need to check the .dockerInit file content. There are two level of comparision we can make: - # * the current container Id with the already intialised one: if we want reinitailisation on every container kill/start - # * the current image Id with the already initialised one: if we want reinitialisation only on docker image rebuild (New WMAgent deployment). - # THE implementation considers the later - reinitialisation on container rebuild - local stepMsg="Performing Docker image to Host initialisation steps" + # This function does all Host modifications needed at Runtime + + local stepMsg="Performing Host initialization steps" echo "-------------------------------------------------------" echo "Start: $stepMsg" - # TODO: This is to be removed once we decide to run it directly from the deploy area - echo "$FUNCNAME: Copy the proper manage file" - cp -fv $WMA_DEPLOY_DIR/bin/manage $WMA_MANAGE_DIR/manage && chmod 755 $WMA_MANAGE_DIR/manage + # TODO: This is to be removed once we decide to run it only from the deploy area + # The only place left where the manage script is called directly from + # $WMA_CONFIG_DIR is from $WMA_DEPLOY_DIR/bin/clean-oracle + echo "$FUNCNAME: Linking the proper manage file from Config Area" + + if [[ -h $WMA_CONFIG_DIR/manage ]] || [[ -f $WMA_CONFIG_DIR/manage ]]; then + rm -f $WMA_CONFIG_DIR/manage + fi + ln -s $WMA_MANAGE_DIR/manage $WMA_CONFIG_DIR/manage echo "$FUNCNAME: Copy the Runtime scripts" - _init_valid $wmaInitRuntime || { - # checking if $WMA_DEPLOY_DIR is root path for $pythonLib: - if [[ $pythonLib =~ ^$WMA_DEPLOY_DIR ]]; then - mkdir -p $WMA_INSTALL_DIR/Docker/ - cp -rav $pythonLib/WMCore/WMRuntime $WMA_INSTALL_DIR/Docker/ - cp -rav $WMA_DEPLOY_DIR/etc/ $WMA_CONFIG_DIR/ - echo $WMA_BUILD_ID > $wmaInitRuntime - else - echo "$FUNCNAME: ERROR: \$WMA_DEPLOY_DIR: $WMA_DEPLOY_DIR is not a root path for \$pythonLib: $pithonLib" - echo "$FUNCNAME: ERROR: We cannot find the correct WMCore/WMRuntime source to copy at the current host!" - return $(false) - fi - } + _init_valid $wmaInitRuntime || _copy_runtime # Check if the host has a basic WMAgent.secrets file and copy a template if missing # NOTE: Here we never overwrite any existing WMAGent.secrets file: We follow: # * Check if there is any at the host, and if so, is it a blank template or a fully configured one - # * In case we find a legit WMAgent.secrets file we set the .dockerInit and move on + # * In case we find a legit WMAgent.secrets file we set the .initAdmin and move on # * In case we need to copy a brand new template (based on the agent type - test/prod ) - # or a blank one found at the host we halt without updating the .dockerInit file + # or a blank one found at the host we halt without updating the .initAdmin file # and we ask the user to examine/update the file. # (Re)Initialization should never pass beyond that step unless properly # configured WMAgent.secrets file being provided at the host. - echo "$FUNCNAME: Initialise && Validate && Load WMAgent.secrets" + echo "$FUNCNAME: Initialize && Validate && Load WMAgent.secrets" _init_valid $wmaInitAdmin || { if [[ ! -f $WMA_SECRETS_FILE ]]; then # NOTE: we consider production templates for relval agents and testbed templates for dev- agents @@ -189,16 +198,9 @@ deploy_to_host(){ echo $WMA_BUILD_ID > $wmaInitRucio } - echo "Done: $stepMsg" - echo "-------------------------------------------------------" - - local stepMsg="Performing local Docker image initialisation steps" - echo "-------------------------------------------------------" - echo "Start: $stepMsg" - - # Checking Certificates and proxy; echo "$FUNCNAME: Checking Certificates and Proxy" _renew_proxy + echo "Done: $stepMsg" echo "-------------------------------------------------------" } @@ -224,8 +226,8 @@ check_wmasecrets(){ _check_oracle() { # Auxiliary function to check if the oracle database configured for the current agent is empty - # NOTE: Oracle is centraly provided - we require an empty database for every account/agent - # otherwise we cannot guarantie this is the only agent to connect to the so configured database + # NOTE: Oracle is centrally provided - we require an empty database for every account/agent + # otherwise we cannot guarantee this is the only agent to connect to the so configured database # NOTE: Here to check if the wmagent database exists and if so to check if it was done from # a container with the current $WMA_BUILD_ID echo "$FUNCNAME: Checking whether the Oracle server is reachable ..." @@ -329,12 +331,33 @@ EOF echo "-----------------------------------------------------------------------" } -check_docker_init() { - # A function to check all previously populated */.dockerInit files - # from all previous steps and compare them with the /data/.dockerBuildId +check_wmagent_init() { + # A function to check all previously populated */.init files + # from all previous steps and compare them with the /data/.wmaBuildId # if all do not match we cannot continue - we consider configuration/version # mismatch between the host and the container + # NOTE: On every step we need to check the .init file content. We always compare + # the current $WMA_BUILD_ID with the one previously initialized at the host. + # The WMA_BUILD_ID is generated during the execution of `install.sh` + # at build time (for docker images) or deploy time for virtual env + # There are few levels of comparison we can make: + # 1. If we want to trigger re-initialization on any WMAgent image rebuild, + # then the $WMA_BUILD_ID should contain a sha256sum of a random variable + # 2. If we want to trigger re-initialization only on new WMAgent tag builds, + # then the $WMA_BUILD_ID should contain a sha256sum of whole $WMA_TAG + # 3. If we want to trigger re-initialization only on release change (not on + # patch version or release candidate change), then we should split + # $WMA_TAG in parts: major, minor and patch(release candidate) part and the + # $WMA_BUILD_ID should contain a sha256sum only of the release = major + minor + # parts excluding the patch version (release candidate) part, e.g.: + # WMA_TAG=2.3.3.1; + # WMA_VER[release]=2.3.3 + # WMA_VER[major]=2.3 + # WMA_VER[minor]=3 + # WMA_VER[patch]=1 + # The current implementation considers option 3 - trigger full initialization only on WMAgent release change + local initFilesList=" $wmaInitAdmin $wmaInitActive @@ -349,20 +372,20 @@ check_docker_init() { $wmaInitRucio $wmaInitUsing " - local stepMsg="Performing checks for successful Docker initialisation steps..." + local stepMsg="Performing checks for successful WMAgent initialisation steps..." echo "-------------------------------------------------------" echo "Start: $stepMsg" - local dockerInitId="" - local dockerInitIdValues="" + local wmaInitId="" + local wmaInitIdValues="" local idValue="" for initFile in $initFilesList; do _init_valid $initFile && idValue=$(cat $initFile 2>&1) || idValue=$initFile - dockerInitIdValues="$dockerInitIdValues $idValue" + wmaInitIdValues="$wmaInitIdValues $idValue" done - dockerInitId=$(for id in $dockerInitIdValues; do echo $id; done |sort|uniq) + wmaInitId=$(for id in $wmaInitIdValues; do echo $id; done |sort|uniq) echo "WMA_BUILD_ID: $WMA_BUILD_ID" - echo "dockerInitId: $dockerInitId" - [[ $dockerInitId == $WMA_BUILD_ID ]] && { echo "OK"; return $(true) ;} || { echo "WARNING: dockerInit vs buildId mismatch"; return $(false) ;} + echo "wmaInitId: $wmaInitId" + [[ $wmaInitId == $WMA_BUILD_ID ]] && { echo "OK"; return $(true) ;} || { echo "WARNING: wmaInitId vs. wmaBuildId mismatch"; return $(false) ;} } activate_agent() { @@ -415,13 +438,17 @@ agent_tweakconfig() { echo "$FUNCNAME: triggered." [[ -f $WMA_CONFIG_DIR/config.py ]] || { echo "ERROR: Missing WMAgent config!"; return $(false) ;} - echo "$FUNCNAME: Making agent configuration changes needed for Docker" - # make this a docker agent - sed -i "s+Agent.isDocker = False+Agent.isDocker = True+" $WMA_CONFIG_DIR/config.py - # update the location of submit.sh for docker - sed -i "s+config.JobSubmitter.submitScript.*+config.JobSubmitter.submitScript = '$WMA_CONFIG_DIR/etc/submit_py3.sh'+" $WMA_CONFIG_DIR/config.py - # replace all tags with current - sed -i "s+$WMA_TAG+current+" $WMA_CONFIG_DIR/config.py + # NOTE: We are not about to change the submit script and runtime sources + # if we are not running from inside a Docker container + _is_docker && { + echo "$FUNCNAME: Making agent configuration changes needed for Docker" + # make this a docker agent + sed -i "s+Agent.isDocker = False+Agent.isDocker = True+" $WMA_CONFIG_DIR/config.py + # update the location of submit.sh for docker + sed -i "s+config.JobSubmitter.submitScript.*+config.JobSubmitter.submitScript = '$WMA_CONFIG_DIR/etc/submit_py3.sh'+" $WMA_CONFIG_DIR/config.py + # replace all tags with current + sed -i "s+$WMA_TAG+current+" $WMA_CONFIG_DIR/config.py + } echo "$FUNCNAME: Making other agent configuration changes" sed -i "s+REPLACE_TEAM_NAME+$TEAMNAME+" $WMA_CONFIG_DIR/config.py @@ -469,6 +496,8 @@ agent_resource_control() { echo "$FUNCNAME: Adding only T1 and T2 sites to resource-control..." manage execute-agent wmagent-resource-control --add-T1s --plugin=SimpleCondorPlugin --pending-slots=50 --running-slots=50 --down ; let errVal+=$? manage execute-agent wmagent-resource-control --add-T2s --plugin=SimpleCondorPlugin --pending-slots=50 --running-slots=50 --down ; let errVal+=$? + elif [[ "$TEAMNAME" == Tier0* ]]; then + echo "$FUNCNAME: Tier0 agent not populating resource control for this agent" else echo "$FUNCNAME: Adding ALL sites to resource-control..." manage execute-agent wmagent-resource-control --add-all-sites --plugin=SimpleCondorPlugin --pending-slots=50 --running-slots=50 --down ; let errVal+=$? @@ -498,18 +527,18 @@ agent_resource_opp() { ## Populating opportunistic resource-control echo "$FUNCNAME: Populating opportunistic resource-control" ## Loop over the env variables starting with RESOURCE_* - for res in ${!RESOURCE_*} + for resRecord in ${!RESOURCE_*} do ## Parsing of the information stored in RESOURCE_* env variable, according to the schema reported above - eval `declare -p $res | sed -e "s/$res/site/g"` - if [[ ${site[name]} =~ .*_US_.* ]] && [[ $HOSTNAME =~ .*cern\.ch ]]; then + eval `declare -p $resRecord | sed -e "s/$resRecord/res/g"` + if [[ ${res[name]} =~ .*_US_.* ]] && [[ $HOSTNAME =~ .*cern\.ch ]]; then echo "I am based at CERN, so I cannot use US opportunistic resources, moving to the next site" continue else - manage execute-agent wmagent-resource-control --plugin=SimpleCondorPlugin --opportunistic --pending-slots=${site[pend]} --running-slots=${site[run]} --add-one-site=${site[name]} ; let errVal+=$? + manage execute-agent wmagent-resource-control --plugin=SimpleCondorPlugin --opportunistic --pending-slots=${res[pend]} --running-slots=${res[run]} --add-one-site=${res[name]} ; let errVal+=$? fi done - [[ $errVal -eq 0 ]] || { echo "ERROR: Failed to populate WMAgent's opportunistic resource control!"; return $(false) ;} + [[ $errVal -eq 0 ]] || { echo "ERROR: Failed to populate WMAgent's opportunistic resource control!"; return $errVal ;} echo $WMA_BUILD_ID > $wmaInitResourceOpp fi @@ -521,25 +550,29 @@ agent_resource_opp() { agent_upload_config(){ # A function to be used for uploading WMAgentConfig to AuxDB at Central CouchDB - # NOTE: The final config/.dockerInit is to be set here after full agent initialisation. + # NOTE: The final config/.initUsing is to be set in the main() function after full agent initialization. # Once we have reached to the step of uploading the agent config to Cerntral CouchDB - # we consider all previous reintialisation and config steps as successfully complpeted. + # we consider all previous re-initialization and config steps as successfully completed. # Steps to follow: - # * Checking if the current image WMA_BUILD_ID matches the last one successfully initialised at the host + # * Checking if the current image WMA_BUILD_ID matches the last one successfully initialized at the host # * If not, tries to upload the current config to Central CouchDB - # * If agent config successfully uploaded, preserve the WMA_BUILD_ID at config/.dockerInit - # With that, we consider the agent initialisation fully complete. The init steps will not be - # repeated on further container restarts unless any of the */.dockerInit files at the host is altered. + # * If agent config successfully uploaded, preserve the WMA_BUILD_ID at config/.initUpload + # With that, we consider the agent initialization fully complete. The init steps will not be + # repeated on further container restarts unless any of the */.init files at the host is altered. local stepMsg="Performing $FUNCNAME" echo "-------------------------------------------------------" echo "Start: $stepMsg" - _init_valid $wmaInitUploaded || { + _init_valid $wmaInitUpload || { echo "$FUNCNAME: triggered." echo "$FUNCNAME: Tweaking central agent configuration befre uploading" if [[ "$TEAMNAME" == production ]]; then echo "$FUNCNAME: Agent connected to the production team, setting it to drain mode" agentExtraConfig='{"UserDrainMode":true}' + elif [[ "$TEAMNAME" == Tier0* ]]; then + echo "$FUNCNAME: Tier0 agent not uploading configuration" + echo $WMA_BUILD_ID > $wmaInitUpload + return elif [[ "$TEAMNAME" == *testbed* ]]; then echo "$FUNCNAME: Testbed agent, setting MaxRetries to 0..." agentExtraConfig='{"MaxRetries":0}' @@ -569,7 +602,7 @@ start_agent() { main(){ basic_checks check_wmasecrets - check_docker_init || { + check_wmagent_init || { (deploy_to_host) || { err=$?; echo "ERROR: deploy_to_host"; exit $err ;} (check_databases) || { err=$?; echo "ERROR: check_databases"; exit $err ;} (activate_agent) || { err=$?; echo "ERROR: activate_agent"; exit $err ;} @@ -579,7 +612,7 @@ main(){ (agent_resource_opp) || { err=$?; echo "ERROR: agent_resource_opp"; exit $err ;} (agent_upload_config) || { err=$?; echo "ERROR: agent_upload_config"; exit $err ;} echo $WMA_BUILD_ID > $wmaInitUsing - (check_docker_init) || { err=$?; echo "ERROR: DockerBuild vs. HostConfiguration version missmatch"; exit $err ; } + (check_wmagent_init) || { err=$?; echo "ERROR: Unresolved wmaInitId vs. wmaBuildId mismatch"; exit $err ; } echo && echo "Docker container has been initialised! However you still need to:" echo " 1) Double check agent configuration: less /data/[dockerMount]/srv/wmagent/current/config/config.py" echo " 2) Start the agent by either of the methods bellow:" @@ -594,6 +627,13 @@ main(){ echo " docker kill wmagent" echo " * Start a fresh instance of wmagent:" echo " ./wmagent-docker-run.sh -t && docker logs -f wmagent" + echo + echo " c) If you are deploying inside a virtual environment" + echo " * Activate the environment:" + echo " cd && . bin/activate" + echo " * Use the regular manage script inside the virtual environment:" + echo " manage start-agent" + echo echo "Have a nice day!" && echo return $(true) } diff --git a/docker/pypi/wmagent/install.sh b/docker/pypi/wmagent/install.sh index ac7a26674..f765414ee 100755 --- a/docker/pypi/wmagent/install.sh +++ b/docker/pypi/wmagent/install.sh @@ -114,9 +114,9 @@ stepMsg="Generating and preserving current build id" echo "-----------------------------------------------------------------------" echo "Start $stepMsg" -echo ${WMA_VER[release]}| sha256sum |awk '{print $1}' > $WMA_ROOT_DIR/.dockerBuildId -echo "WMA_BUILD_ID:`cat $WMA_ROOT_DIR/.dockerBuildId`" -echo "WMA_BUILD_ID preserved at: $WMA_ROOT_DIR/.dockerBuildId " +echo ${WMA_VER[release]} | sha256sum | awk '{print $1}' > $WMA_ROOT_DIR/.wmaBuildId +echo "WMA_BUILD_ID:`cat $WMA_ROOT_DIR/.wmaBuildId`" +echo "WMA_BUILD_ID preserved at: $WMA_ROOT_DIR/.wmaBuildId " echo "Done $stepMsg!" && echo echo "-----------------------------------------------------------------------" @@ -144,7 +144,7 @@ tweakEnv(){ cat <> $WMA_ENV_FILE -export WMA_BUILD_ID=\$(cat \$WMA_ROOT_DIR/.dockerBuildId) +export WMA_BUILD_ID=\$(cat \$WMA_ROOT_DIR/.wmaBuildId) export WMCORE_ROOT=\$WMA_DEPLOY_DIR export WMAGENT_CONFIG=\$WMA_CONFIG_FILE export WMAGENTPY3_ROOT=\$WMA_INSTALL_DIR @@ -159,7 +159,7 @@ EOF } -stepMsg="Tweaking runtime environment for user: $WMA_USER" +stepMsg="Tweaking runtime environment." echo "-----------------------------------------------------------------------" echo "Start $stepMsg" tweakEnv || { err=$?; echo ""; exit $err ; }