diplodocus.org Git - nmh/blob - docs/imap/testing/run

   1 #!/usr/local/bin/zsh
   2
   3 # Quick-n-dirty script to ease running the IMAP tests in various configurations.
   4 # Also generates coverage reports (requires Rust nightly).
   5 # TODO redo in more standard fashion under `make check`.
   6
   7 # Configuration:
   8 # export MH_TEST_IMAP_LOG_LEVEL=trace
   9 # export MH_TEST_NOCLEANUP=1
  10 # export MH_TEST_STUNNEL_DEBUG=7
  11 # export MH_TEST_STUNNEL_OUTPUT=/dev/stderr
  12 # export MH_TEST_VERBOSE=1
  13 # export RUST_BACKTRACE=1
  14 # abs_srcdir=$HOME/work/nmh
  15 # abs_builddir=$abs_srcdir/.o/clang13
  16 # LLVM_PROFDATA=llvm-profdata13
  17 # LLVM_COV=llvm-cov13
  18 # MAKE=make
  19 # MAKE_JOBS=-j4
  20 # PANDA_IMAPD=/usr/local/libexec/imapd
  21 # # optional:
  22 # gandi_user='example@example.com'
  23 # gandi_pw='...'
  24 #gmail_user='example@example.com'
  25 #gmail_oauth=/path/to/oauth-gmail
  26
  27 set -ex
  28
  29 abs_builddir=$PWD
  30
  31 cf=$1
  32 if ! [[ -n $cf ]]; then
  33     print -u2 "usage imaptest CONFIGURATION ACTION"
  34     exit 2
  35 fi
  36 shift
  37
  38 . $cf
  39
  40 for v in \
  41         abs_srcdir \
  42         LLVM_PROFDATA \
  43         LLVM_COV \
  44         MAKE \
  45         PANDA_IMAPD \
  46         ; do
  47     if ! eval "[[ -n \$$v ]]"; then
  48         print -u2 "$v must be set in $cf"
  49         exit 2
  50     fi
  51 done
  52
  53 testdir=$abs_builddir/test/testdir
  54
  55 export CARGO_TARGET_DIR=$abs_builddir/target
  56
  57 # Coverage documentation
  58 # https://doc.rust-lang.org/beta/unstable-book/compiler-flags/source-based-code-coverage.html
  59 # https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
  60 # https://llvm.org/docs/CommandGuide/llvm-profdata.html
  61 # https://llvm.org/docs/CommandGuide/llvm-cov.html
  62 # Bugs
  63 # https://github.com/rust-lang/rust/issues/79121 Tracking Issue for LLVM InstrProf code coverage
  64 # https://github.com/rust-lang/rust/issues/85902 Add a cargo option to generate coverage reports using -Z instrument-coverage
  65 # https://github.com/rust-lang/rust/issues/79649 Add support for branch coverage in source-based coverage
  66 report-coverage() {
  67     typeset profile=$1
  68     typeset profile_dir=$2
  69     typeset profdata=$abs_builddir/covdata/$profile.profdata
  70     find $abs_builddir/covdata/$profile_dir -name '*.profraw' | $nop xargs $LLVM_PROFDATA merge --sparse \
  71                                                                       -o $profdata
  72     $nop $LLVM_COV show -Xdemangler=$HOME/.cargo/bin/rustfilt '--ignore-filename-regex=/tests/' '--ignore-filename-regex=/tests\.rs' '--ignore-filename-regex=\.cargo' --format=html \
  73                -o $abs_builddir/covdata/html/$profile --instr-profile=$profdata \
  74                ${=${(j: --object :)test_objects}}
  75 }
  76
  77 check-litter() {
  78     setopt LOCAL_OPTIONS NULL_GLOB
  79     typeset litter=($testdir/*.actual)
  80     (( $#litter > 0 )) || return 0
  81     print -u2 "$1 littered $litter"
  82     false
  83 }
  84
  85 report_only=false
  86 coverage=false
  87 typeset -a actions profiles
  88 while [[ $# -gt 0 ]]; do
  89     word=$1
  90     shift
  91     case $word in
  92         -n)
  93             nop=(print --)
  94             ;;
  95
  96         -cover*)
  97             coverage=:
  98             if [[ $word = -cover-report ]]; then
  99                 report_only=:
 100             else
 101                 $nop rm -rf $abs_builddir/covdata
 102             fi
 103             . ~/.cargo/env
 104             export RUSTFLAGS="-Zinstrument-coverage $RUSTFLAGS"
 105             ;;
 106
 107         all)
 108             # We test implicit TLS with Gmail and explicit with Gandi; any reason to test both ways both places?
 109             set -- cargot test live/panda live/dovecot live/cyrus live/gandi-starttls live/gmail "$@"
 110             ;;
 111
 112         build|clean)
 113             actions+=$word
 114             ;;
 115
 116         cargot|test|live/*)
 117             actions+=$word
 118             profiles+=$word
 119             ;;
 120
 121         test/*)
 122             actions+=$word
 123             profiles+=test
 124             ;;
 125
 126         *)
 127             print -u2 "get outta here $word"
 128             exit 3
 129             ;;
 130     esac
 131 done
 132
 133 if ! $report_only; then
 134 for action in $actions; do
 135     case $action in
 136         build)
 137             (cd $abs_builddir && $nop $MAKE $MAKE_JOBS all cargo)
 138             ;;
 139
 140         cargot)
 141             $nop env LLVM_PROFILE_FILE="$abs_builddir/covdata/cargot/%m-%p.profraw" cargo t -- --test-threads=1
 142             ;;
 143
 144         test)
 145             tests=(
 146                 test/imap/test-astring
 147                 test/imap/test-bad
 148                 test/imap/test-bad-fetch-body
 149                 test/imap/test-folder-create-disallowed
 150                 # Downloads all mail from your test account; better be sure you want that!
 151                 # test/imap/test-gmail-all
 152                 test/imap/test-inc
 153                 test/imap/test-inc-missing-msg
 154                 test/imap/test-inc-missing-msg-case-2
 155                 # Handled specially below.
 156                 # test/imap/test-live
 157                 test/imap/test-login
 158                 test/imap/test-mark
 159                 test/imap/test-quirks-gmail
 160                 # TODO fix the bitrot
 161                 # test/imap/test-refile
 162                 test/imap/test-rmm
 163                 test/imap/test-scan
 164                 test/imap/test-server-drops
 165                 test/imap/test-uidvalidity
 166                 test/imap/test-xoauth
 167             )
 168             (cd $abs_builddir && for test in $tests; do
 169                  $nop env PANDA_IMAPD=$PANDA_IMAPD MH_COVERAGE_PROFILE=test $MAKE check TESTS=$test
 170                  check-litter $test
 171              done)
 172         ;;
 173
 174         test/*)
 175             (cd $abs_builddir && $nop env PANDA_IMAPD=$PANDA_IMAPD MH_COVERAGE_PROFILE=test $MAKE check TESTS=$action)
 176             check-litter $action
 177         ;;
 178
 179         live/panda)
 180             (cd $abs_builddir && $nop env PANDA_IMAPD=$PANDA_IMAPD MH_COVERAGE_PROFILE=live/panda MH_IMAP_TEST_SERVER=panda \
 181                  $MAKE check TESTS=test/imap/test-live)
 182             check-litter $action
 183             ;;
 184
 185         live/dovecot)
 186             (cd $abs_builddir && $nop env MH_COVERAGE_PROFILE=live/dovecot MH_IMAP_TEST_SERVER=127.0.0.1 MH_IMAP_TEST_PORT=1043 \
 187                  MH_IMAP_TEST_USER=test MH_IMAP_TEST_PASSWORD=testpw MH_IMAP_TEST_EMAIL_ADDRESS=test MH_IMAP_TEST_TLS=no \
 188                  MH_IMAP_SERVER_DRAFT_BORN_UNSEEN=' ' MH_IMAP_SERVER_EATS_TRAILING_EOL=' ' \
 189                  $MAKE check TESTS=test/imap/test-live)
 190             check-litter $action
 191             ;;
 192
 193         live/cyrus)
 194             (cd $abs_builddir && $nop env MH_COVERAGE_PROFILE=live/cyrus MH_IMAP_TEST_SERVER=127.0.0.1 MH_IMAP_TEST_PORT=143 MH_IMAP_TEST_USER=test \
 195                  MH_IMAP_TEST_PASSWORD=test MH_IMAP_TEST_EMAIL_ADDRESS=test MH_IMAP_TEST_TLS=no MH_IMAP_TEST_MAILBOX_ROOT=test/ \
 196                  MH_IMAP_SERVER_DRAFT_BORN_UNSEEN='U' \
 197                  $MAKE check TESTS=test/imap/test-live)
 198             check-litter $action
 199             ;;
 200
 201         # Looks like Gandi use Dovecot
 202         live/gandi)
 203             (cd $abs_builddir && $nop env MH_COVERAGE_PROFILE=live/gandi MH_IMAP_TEST_SERVER=mail.gandi.net MH_IMAP_TEST_PORT=imaps \
 204                  MH_IMAP_TEST_USER=$gandi_user MH_IMAP_TEST_PASSWORD=$gandi_pw MH_IMAP_TEST_EMAIL_ADDRESS=$gandi_user \
 205                  MH_IMAP_TEST_TLS=implicit MH_IMAP_TEST_MAILBOX_ROOT=test/ \
 206                  MH_IMAP_SERVER_DRAFT_BORN_UNSEEN=' '  \
 207                  $MAKE check TESTS=test/imap/test-live)
 208             check-litter $action
 209             ;;
 210
 211         live/gandi-starttls)
 212             (cd $abs_builddir && $nop env MH_COVERAGE_PROFILE=live/gandi-starttls MH_IMAP_TEST_SERVER=mail.gandi.net MH_IMAP_TEST_PORT=imap \
 213                  MH_IMAP_TEST_USER=$gandi_user MH_IMAP_TEST_PASSWORD=$gandi_pw MH_IMAP_TEST_EMAIL_ADDRESS=$gandi_user \
 214                  MH_IMAP_TEST_TLS=explicit MH_IMAP_TEST_MAILBOX_ROOT=test/ \
 215                  MH_IMAP_SERVER_DRAFT_BORN_UNSEEN=' '  \
 216                  $MAKE check TESTS=test/imap/test-live)
 217             check-litter $action
 218             ;;
 219
 220         live/gmail)
 221             (cd $abs_builddir && $nop env MH_COVERAGE_PROFILE=live/gmail MH_IMAP_TEST_SERVER=gmail MH_IMAP_TEST_PORT=imaps MH_IMAP_TEST_USER=$gmail_user \
 222                  MH_IMAP_TEST_OAUTH_FILE=$gmail_oauth MH_IMAP_TEST_EMAIL_ADDRESS=$gmail_user MH_IMAP_TEST_MAILBOX_ROOT=test/ \
 223                  MH_IMAP_SERVER_DRAFT_BORN_UNSEEN='U'  \
 224                  $MAKE check TESTS=test/imap/test-live)
 225             check-litter $action
 226             ;;
 227
 228         report-coverage)
 229             ;;
 230
 231         clean)
 232             $nop rm -rf $abs_builddir/target
 233             $nop rm -rf $testdir
 234             ;;
 235     esac
 236 done
 237 fi
 238
 239 if $coverage; then
 240     test_objects=(
 241         $abs_builddir/target/debug/imap-folder-manager
 242         $($nop env LLVM_PROFILE_FILE="$abs_builddir/covdata/cargot/%m-%p.profraw" cargo t --tests --no-run --message-format=json 2>/dev/null \
 243                 | $nop perl -MJSON::PP -lne 'eval { $item = decode_json($_) }; if ($@) { @all=(); die($@) } $fns = $item->{profile}->{test} && $item->{filenames}; $fns && push(@all, @$fns); END { print join("\n", @all) }')
 244     )
 245     print -l $test_objects
 246     cd $abs_srcdir
 247     report-coverage all
 248     (( ${profiles[(i)live/*]} <= ${#profiles} )) && report-coverage live live
 249     for profile in $profiles; do
 250         report-coverage $profile $profile
 251     done
 252     $nop chmod -R a+rX $abs_builddir/covdata/html
 253     exit
 254 fi