#!/usr/bin/env bash # # SPDX-License-Identifier: GPL-2.0-only # This script creates a list of commits between two releases, broken out into # fairly inexact categories, based on the directories that files are in. If # a commit touched anything in the path that is checked earlier, it's counted # as being in that category. # # Don't run this in your current working tree, it checks out different versions # and can lose things. # set -x # uncomment for debug # Check for tools if ! ( git --version && cloc --version && rename --version ) > /dev/null 2>&1 then echo "ERROR: cloc, git or rename is not installed. Exiting" exit 1 fi if ! { cdup="$(git rev-parse --show-cdup 2>/dev/null)" && [ -z "${cdup}" ]; } then echo "ERROR: This is not the top directory of a git repo. Exiting." exit 1 fi # Try to verify that the repo is clean before losing state. if ! git diff-index --quiet --cached HEAD 2>/dev/null; then echo "ERROR: repo is not clean. Exiting." exit 1 fi # Verify the command line arguments if [ "$1" == "--help" ] || [ -z "$1" ] || [ -z "$2" ]; then echo echo "Usage: $0 <old_version> <new_version> [release notes file]" echo "Old version should be a tag (4.1), a branch (origin/4.1), or a commit id" echo "New version can be 'HEAD' a branch (origin/master) a tag (4.2), or a commit id" echo "Logfile can be a new file or an existing file to update" echo "Example: \"$0 origin/4.1 4.2 rnotes.txt\"" echo echo "Note that the script starts at the commit AFTER the old version." echo exit 1 else OLD_GIT_VERSION="$1" NEW_GIT_VERSION="$2" if [ "$OLD_GIT_VERSION" = "HEAD" -o "$NEW_GIT_VERSION" = "HEAD" ]; then echo "Error: using HEAD as a reference doesn't work" echo exit 1 fi TOTAL_COMMITS=$(git log --pretty=oneline \ "${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | wc -l) fi TOP=$(pwd) if [ -n "$3" ]; then MAIN_LOGFILE="${TOP}/$3" else MAIN_LOGFILE="${TOP}/relnotes.txt" fi # Figure out which logfile we're writing to. If the specified logfile exists, # we need to write to a temporary logfile, then append changes to the main # logfile. if [ -f "$MAIN_LOGFILE" ]; then LOGFILE="$(mktemp "LOGFILE.XXXX")" LOGFILE="${TOP}/$LOGFILE" UPDATE_MAIN_LOGFILE=1 else LOGFILE="$MAIN_LOGFILE" fi get_author_commit_count() { git log "${NEW_GIT_VERSION}" 2>/dev/null | grep -c "^Author: $1" } # Print and log the versions log_versions() { echo "Log of commit $1 to commit $2" echo "Log of commit $1 to commit $2" >> "$LOGFILE" echo "Total commits: ${TOTAL_COMMITS}" echo "Total commits: ${TOTAL_COMMITS}" >> "$LOGFILE" echo } # Get the first commit id in the current tree get_latest_commit_id() { ( cd "$1" git log 2>/dev/null | grep '^commit ' | head -1 | sed 's/commit //' ) } # Main get log function _get_log() { local oldver="$1" local newver="$2" local title="$3" local paths="$4" local keywords="$5" # Leave ${paths} unquoted in the git commands # shellcheck disable=SC2086 { \ if [ -n "$paths" ]; then \ git log --abbrev-commit --pretty=oneline \ "${oldver}..${newver}" -- ${paths} \ 2>/dev/null; \ fi; \ if [ -n "$keywords" ]; then \ git log --abbrev-commit --pretty=oneline \ "${oldver}..${newver}" 2>/dev/null \ | grep -i "$keywords"; \ fi \ } | sort -t ' ' -k 2 | uniq } # Output to a new log, then compare to the first logfile, and only output # non duplicated lines to the final file. get_log_dedupe() { local title="$1" local paths="$2" local keywords="$3" local log local commits dedupe_tmpfile="$(mktemp "LOGFILE.XXXX")" log=$(_get_log "$OLD_GIT_VERSION" "$NEW_GIT_VERSION" \ "$title" "$paths" "$keywords") echo "$log" > "$dedupe_tmpfile" log=$(grep -Fxv -f "$LOGFILE" "$dedupe_tmpfile") commits=$(echo "$log" | wc -l) #echo "$title: $paths $keywords" >> "$LOGFILE" printf "%s\n%s\n\n" "##### $title ($commits commits) #####" \ "$log" >> "$LOGFILE" rm "$dedupe_tmpfile" } # get logs for the submodules get_log_submodule() { local old_version="$1" local new_version="$2" local submodule_dir="$3" local log local commits printf "Submodule %s\n" "$submodule_dir" printf "commit %s to commit %s\n\n" "$old_version" "$new_version" ( cd "${TOP}/$submodule_dir" log="$(_get_log "$old_version" "$new_version" "$submodule_dir" "." "")" commits=$(echo "$log" | wc -l) if [ -n "$log" ]; then printf "%s\n" "$submodule_dir ($commits commits)" >> "$LOGFILE" printf "\n" >> "$LOGFILE" fi ) } find_areas() { local directory="$1" local filename="$2" local skip="$3" find "$directory" -name "$filename" | sed "s|/$filename||" | sed "s|/$directory||" | grep -v "$skip" | sort } # Make sure things get cleaned up if ctl-c is pressed while the old version # is checked out and files are renamed. This can be a real mess to clean # up manually. version_ctrl_c() { printf "\n** Trapped CTRL-C\n Cleaning up and exiting.\n" find 'src' -name 'gnumakefile' \ -exec rename 's/gnumakefile/Makefile\.inc/' {} \; git checkout origin/master > /dev/null 2>&1 git submodule update --init --checkout > /dev/null 2>&1 rm "$LOGFILE" exit 1; } # Calculate areas that have been added or removed based on file lists show_diff () { local new local old new="$(comm -13 <(echo "$2") <(echo "$3"))" old="$(comm -23 <(echo "$2") <(echo "$3"))" # Allow running a postprocessor, given as 4th argument over the # resulting diff, provide context if it's old or new data if [ -n "$4" ]; then new=$(echo "$new" | $4 new | sort) old=$(echo "$old" | $4 old | sort) fi new="$(printf "$new" | sed 's/^/* /')" old="$(printf "$old" | sed 's/^/* /')" if [ -n "$new" ]; then printf "Added %s $1:\n-------------------\n%s\n\n" \ "$(echo "$new" | wc -l)" "$new" >> "$LOGFILE" fi if [ -n "$old" ]; then printf "Removed %s $1:\n-------------------\n%s\n\n" \ "$(echo "$old" | wc -l)" "$old" >> "$LOGFILE" fi } get_sloc () { # Because cloc works on extensions, and .inc identifies as pascal, # while we use it both for Makefile.inc and some files that are # really C, do three passes: everything but .inc files, all .inc files # that aren't Makefiles, all Makefile.inc, then combine them. local base=`mktemp` find src -name Makefile.inc > ${base}.mak cloc --progress-rate=0 --quiet \ --script-lang="Bourne Shell",bash --exclude-ext=inc \ --exclude-dir=vendorcode --out=${base} src cloc --progress-rate=0 --quiet \ --exclude-list-file=${base}.mak --force-lang=c,inc \ --exclude-dir=vendorcode --out=${base}.c src cloc --progress-rate=0 --quiet \ --list-file=${base}.mak --force-lang=make,inc \ --exclude-dir=vendorcode --out=${base}.m src cloc --progress-rate=0 --quiet --sum-reports \ ${base} ${base}.c ${base}.m --out ${base}.result echo cat ${base}.result.lang rm -f ${base}* } # Start collecting data from the old and new revisions. # This is relatively disruptive to the tree, so trap on ctl-c so that # things can be put back to normal trap version_ctrl_c SIGINT #check out old version and get information printf -- "Finding old submodule versions...\n" git checkout "$OLD_GIT_VERSION" > /dev/null 2>&1 git submodule update --init --checkout > /dev/null 2>&1 for module in $(git submodule --quiet foreach pwd); do name="$(basename "$module" | sed 's/-/_/g')" version="${name^^}_OLD_VERSION" declare "$version"=$(get_latest_commit_id "$module") done printf "Logging directories in the old tree\n" mainboard_list_old=$(grep -h "^[[:space:]]*config\>[[:space:]]*\<BOARD_" src/mainboard/*/*/Kconfig.name 2>/dev/null | sed "s,^.*\<BOARD_\([A-Z0-9_]*\)\>.*$,\1," | sort) cpu_list_old=$(find_areas "src/cpu" "Kconfig" "intel/common") soc_list_old=$(find_areas "src/soc" "Makefile.inc" "intel/common\|amd/common\|romstage") northbridge_list_old=$(find_areas "src/northbridge" "Kconfig" "") sio_list_old=$(find_areas "src/superio" "Makefile.inc" "") southbridge_list_old=$(find_areas "src/southbridge" "Kconfig" "") printf "Calculating old SLOC\n" OLD_SLOC=$(get_sloc) #check out new version and get information printf -- "\nFinding new submodule versions...\n" git checkout "$NEW_GIT_VERSION" > /dev/null 2>&1 git submodule update --init --checkout > /dev/null 2>&1 for module in $(git submodule --quiet foreach pwd); do name="$(basename "$module" | sed 's/-/_/g')" version="${name^^}_NEW_VERSION" declare "$version"=$(get_latest_commit_id "$module") done printf "Logging directories in the new tree\n" mainboard_list_new=$(grep -h "^[[:space:]]*config\>[[:space:]]*\<BOARD_" src/mainboard/*/*/Kconfig.name 2>/dev/null | sed "s,^.*\<BOARD_\([A-Z0-9_]*\)\>.*$,\1," | sort) cpu_list_new=$(find_areas "src/cpu" "Kconfig" "intel/common") soc_list_new=$(find_areas "src/soc" "Makefile.inc" "intel/common\|amd/common\|romstage") northbridge_list_new=$(find_areas "src/northbridge" "Kconfig" "") sio_list_new=$(find_areas "src/superio" "Makefile.inc" "") southbridge_list_new=$(find_areas "src/southbridge" "Kconfig" "") printf "Calculating new SLOC\n" NEW_SLOC=$(get_sloc) git checkout origin/master > /dev/null 2>&1 git submodule update --init --checkout > /dev/null 2>&1 trap "" SIGINT # Done collecting data from the old and new versions # Start outputting to logfile echo "Generating release notes from version ${OLD_GIT_VERSION} to ${NEW_GIT_VERSION}" echo; echo "Main repo" echo "Main repo" >> "$LOGFILE" echo "------------------" >> "$LOGFILE" log_versions "$(git log --pretty=%H \ "${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | tail -1)" \ "$(git log --pretty=%H \ "${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | head -1 )" echo "" >> "$LOGFILE" ### SPECIFIC AREAS FOR RELEASE ### get_log_dedupe "cleanup" "" "spelling\|Use tabs\|transition away from device_t\|space [around\|before]\|code formatting\|commented code\|code cleanup\|capitalize\|unnecessary whitespace\|checkpatch\|remove unused\|remove.*not used" get_log_dedupe "Google Kahlee / AMD Gardenia" "src/mainboard/google/kahlee src/mainboard/amd/gardenia" "kahlee\|gardenia" get_log_dedupe "AMD Stoney Ridge" "src/soc/amd/stoneyridge" "stoney" get_log_dedupe "Google Eve / Poppy / Fizz / Soraka / Nautilus / Intel KblRvp" "src/mainboard/google/eve src/mainboard/google/poppy src/mainboard/google/fizz src/mainboard/intel/kblrvp" "eve[ :]\|poppy\|fizz\|soraka\|nautilus\|kblrvp" #get_log_dedupe "Intel Kunimitsu / Google Chell / Lars / Glados" "src/mainboard/google/lars src/mainboard/google/chell src/mainboard/google/glados src/mainboard/intel/kunimitsu" "chell\|lars\|kunimitsu" get_log_dedupe "Purism SKL" "src/mainboard/purism/librem_skl" "librem13v2\|librem_skl" get_log_dedupe "Intel Skylake / Kabylake" "src/soc/intel/skylake" "skylake\|sky.lake\|kabylake\|kaby.lake" get_log_dedupe "Google Cyan / Intel Strago" "src/mainboard/google/cyan src/mainboard/intel/strago" "cyan\|strago" get_log_dedupe "Intel Braswell" "src/soc/intel/braswell" "braswell" get_log_dedupe "Google Kevin / Gru / Bob / Scarlet / Nefario" "src/mainboard/google/gru" "kevin\|gru[^b]" get_log_dedupe "Rockchip rk3399" "src/soc/rockchip/rk3399" "rk3399" get_log_dedupe "Google Reef / Pyro / Sand / Snappy / Nasher" "src/mainboard/google/reef" "reef\|pyro\|sand[ :]\|snappy\|nasher" #get_log_dedupe "Intel apollolake_rvp leafhill minnow3" "src/mainboard/intel/apollolake_rvp src/mainboard/intel/leafhill src/mainboard/intel/minnow3" "apollolake_rvp\|leafhill\|minnow3" get_log_dedupe "Siemens mc_apl1" "src/mainboard/siemens/mc_apl1" "mc_apl1" get_log_dedupe "Intel Apollolake" "src/soc/intel/apollolake" "apollolake\|apollo.lake" get_log_dedupe "Intel CannonLake" "src/soc/intel/cannonlake src/mainboard/intel/cannonlake_rvp" "cannonlake" get_log_dedupe "Intel Galileo" "src/mainboard/intel/galileo" "galileo" get_log_dedupe "Intel Quark" "src/soc/intel/quark" "quark" get_log_dedupe "Intel Baytrail" "src/soc/intel/baytrail" "baytrail" #get_log_dedupe "Google Gale / Qualcomm QX ipq40xx" "src/mainboard/google/gale src/soc/qualcomm/ipq40xx" "gale\|ipq40" #et_log_dedupe "Google Rotor / Marvell Mvmap2315" "src/soc/marvell/mvmap2315 src/mainboard/google/rotor" "marvell\|mvmap\|rotor" get_log_dedupe "Gigabyte ga-g41m-es2l" "src/mainboard/gigabyte/ga-g41m-es2l" "es2l" get_log_dedupe "Intel x4x northbridge / LGA775" "src/northbridge/intel/x4x src/cpu/intel/socket_LGA775" "x4x\|lga775" get_log_dedupe "Intel Sandybridge / Ivybridge" "src/southbridge/intel/bd82x6x src/southbridge/intel/fsp_bd82x6x src/northbridge/intel/sandybridge src/northbridge/intel/fsp_sandybridge src/cpu/intel/fsp_model_206ax src/cpu/intel/model_206ax" "sandybridge\|ivybridge\|bd82x6x" get_log_dedupe "Intel Common" "src/soc/intel/common src/southbridge/intel/common src/northbridge/intel/common" "" get_log_dedupe "Amd Common" "src/soc/amd/common src/southbridge/amd/common" "" get_log_dedupe "Nvidia" "src/southbridge/nvidia" "" get_log_dedupe "Via" "src/northbridge/via src/southbridge/via" "" get_log_dedupe "Broadcom" "src/southbridge/broadcom" "" get_log_dedupe "Qualcomm" "src/soc/qualcomm" "" get_log_dedupe "Intel vendorcode / FSP" "src/drivers/intel/fsp* src/vendorcode/intel" "" get_log_dedupe "AMD vendorcode / AGESA / PI" "src/vendorcode/amd" "" get_log_dedupe "Google vendorcode" "src/vendorcode/google" get_log_dedupe "TPM" "src/drivers/i2c/tpm src/security/tpm" "tpm" get_log_dedupe "Vboot" "src/vboot" "vboot" ### GENERAL AREAS FOR RELEASE ### # shellcheck disable=SC2013 { get_log_dedupe "ARM" \ "$(for codedir in $(grep -rl "_ARM" --include=Kconfig | \ grep -v 'src/mainboard\|payloads/\|drivers/\|vendorcode/\|console' ); \ do dirname "$codedir"; done | grep -v '^src$')" get_log_dedupe "RISC-V" \ "$(for codedir in $(grep -rl "_RISCV" --include=Kconfig | grep -v 'payloads/\|drivers/\|vendorcode/\|console' ); do dirname "$codedir"; done | grep -v '^src$')" \ "riscv\|risc-v\|sifive" } get_log_dedupe "X86 intel" \ "src/cpu/intel src/soc/intel src/northbridge/intel \ src/southbridge/intel src/include/intel src/drivers/intel" get_log_dedupe "X86 amd" \ "src/cpu/amd src/northbridge/amd src/southbridge/amd src/include/amd src/soc/amd" \ "agesa\|binarypi\|binary.pi" get_log_dedupe "X86 common" \ "src/arch/x86 src/cpu/x86 src/include/x86 src/include/pc80" get_log_dedupe "Mainboards" "src/mainboard/" # Next, print all the rest of the specific areas get_log_dedupe "ACPI" "src/acpi/" get_log_dedupe "Console" "src/console src/include/console" get_log_dedupe "SuperIO" "src/superio src/include/superio" get_log_dedupe "EC" "src/ec" get_log_dedupe "Drivers" "src/drivers" get_log_dedupe "Devices" "src/device src/include/device" # 5th, print the generic areas - This goes late so that the specific # area changes will catch any commits in these areas first. get_log_dedupe "Toolchain" "util/crossgcc" get_log_dedupe "cbfstool" "util/cbfstool" get_log_dedupe "Lint tools" "util/lint" get_log_dedupe "Lib" "src/lib" get_log_dedupe "Commonlib" "src/commonlib" get_log_dedupe "Include" "src/include" get_log_dedupe "Utilities" "util" get_log_dedupe "Payloads" "payloads" get_log_dedupe "Vendorcode" "src/vendorcode" get_log_dedupe "Documentation" "Documentation README" # Then look at areas that are usually outside the mainboards and architectures get_log_dedupe "Build system" \ "Makefile Makefile.inc toolchain.inc src/Kconfig src/cpu/Makefile.inc" get_log_dedupe "Submodules" "3rdparty util/nvidia/cbootimage" get_log_dedupe "Maintainers" "MAINTAINERS" "" # Finally, get anything that was missed above get_log_dedupe "MISC" "." # Replace VENDOR_DEVICE from stdin with their nice names on stdout real_mainboard_names() { local tree_version=$1 # "old" or "new" local git_version_var=${tree_version^^}_GIT_VERSION local git_version=${!git_version_var} local line while read line; do local file="$(git grep -l "^[[:space:]]*config\>[[:space:]]*\<BOARD_${line}\$" ${git_version} -- src/mainboard/\*/\*/Kconfig.name | sed "s,^${git_version}:,,")" if [ -z "$file" ]; then echo "This shouldn't happen: couldn't find Kconfig for $line" exit 1 fi local vendor="$(git grep \" ${git_version} -- $(echo ${file} | cut -d/ -f1-3,5) | cut -d\" -f2)" git grep -h -A1 "^[[:space:]]*config\>[[:space:]]*\<BOARD_${line}\$" ${git_version} -- ${file} | grep \" |cut -d\" -f2 |sed "s,^,${vendor} ," done } # Show areas that have been added or removed show_diff "mainboards" "$mainboard_list_old" "$mainboard_list_new" real_mainboard_names show_diff "processors" "$cpu_list_old" "$cpu_list_new" show_diff "socs" "$soc_list_old" "$soc_list_new" show_diff "northbridges" "$northbridge_list_old" "$northbridge_list_new" show_diff "southbridges" "$southbridge_list_old" "$southbridge_list_new" show_diff "sios" "$sio_list_old" "$sio_list_new" # Log submodules printf "Submodules\n----------\n" >> "$LOGFILE" for module in $(git submodule --quiet foreach pwd); do name=$(basename "$module") # shellcheck disable=SC2001 path=${module#$PWD} old_version=$(echo "${name^^}_OLD_VERSION" | sed 's/-/_/g') new_version=$(echo "${name^^}_NEW_VERSION" | sed 's/-/_/g') get_log_submodule "${!old_version}" "${!new_version}" "$path" done printf "\nrepo statistics\n-------------------\n" >> "$LOGFILE" before_names="$(mktemp "OLDNAMES.XXXX")" after_names="$(mktemp "NEWNAMES.XXXX")" NEW_AUTHORS=$(git log --pretty=%an "${OLD_GIT_VERSION}" 2>/dev/null | sort | \ uniq > "$before_names" && \ git log --pretty=%an "${NEW_GIT_VERSION}" 2>/dev/null | \ sort | uniq > "$after_names" && \ grep -Fxv -c -f "$before_names" "$after_names") NEW_AUTHOR_LIST=$( grep -Fxv -f "$before_names" "$after_names") rm -f "$before_names" "$after_names" { printf -- "- Total commits: %s\n" "$TOTAL_COMMITS" printf -- "- Total authors: %s\n" \ "$(git log "${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | \ grep -e '^Author:' | sed 's/.*Author: //' | sed 's/ <.*.>//' | \ sort | uniq | wc -l)" printf -- "- New authors: %s\n\nNew Authors:\n%s\n" "$NEW_AUTHORS" \ "$NEW_AUTHOR_LIST" } >> "$LOGFILE" printf "Getting developer list\n" printf "\n%-40s: %5s\n" "Developer" "Commits" >> "$LOGFILE" git log "${OLD_GIT_VERSION}..${NEW_GIT_VERSION}" 2>/dev/null | grep '^Author: ' | \ sed 's|Author: ||' | sed 's|\s<.*||' | sort | uniq | \ while read -r line; do printf "%-40s: %5s %5s\n" "$line" \ "$(git log "${OLD_GIT_VERSION}" 2>/dev/null | \ grep -c "^Author: ${line} <")" \ "$(git log "${NEW_GIT_VERSION}" 2>/dev/null | \ grep -c "^Author: ${line} <")" >> "$LOGFILE"; done printf "\nOld SLOC (%s)\n%s" "$NOW" "$OLD_SLOC" >> "$LOGFILE" printf "\nNew SLOC (%s)\n%s" "$NOW" "$NEW_SLOC" >> "$LOGFILE" # Add the collected data to the top of the existing logfile for parsing if [ -n "$UPDATE_MAIN_LOGFILE" ]; then tmpfile="$(mktemp "LOGFILE.XXXX")" grep -Fxv -f "$MAIN_LOGFILE" "$LOGFILE" > "$tmpfile" printf "\n\n" >> "$tmpfile" cat "$MAIN_LOGFILE" >> "$tmpfile" mv "$tmpfile" "$MAIN_LOGFILE" rm -f "$LOGFILE" fi printf "Done.\n"