extract_utils: introduce support for executing blob fixups
* Traditionally, the task of hex-editing blobs has been approached in 2 ways:
(1) Do it out-of-band, commit the modified blob, and record its edited
sha1sum in proprietary-files.txt (aka pin it).
(2) Do it in-band, by adding code to the device-level extract-files.sh
(usually this performs patchelf or sed). This code runs after the
extract_utils functions were invoked.
* Problems of approach (1):
- It relies on verbal (basically commit message) documentation of
the hex-editing that was done. Makes it more difficult to reproduce.
- Each time blobs are updated, pinning needs to be temporarily removed,
hex-editing done again manually and new hash put back.
* Problems of approach (2):
- It is incompatible with the concept of pinning, which is useful
for kanging blobs from another device. A pinned blob would either:
- Match the hash, get hex-edited, then it won't match the hash
next time around.
- Not match the hash (because of, say, hex-editing), then the
extraction script would use an unwanted blob version instead of the
pinned one (either that, or say "!! file not found in source").
* In summary, this patch adds system-wide support for approach (2) in order
to address the aforementioned shortcomings.
* At device level, users of extract_utils who wish to perform blob
fixups can override a blob_fixup() Bash function in their
extract-files.sh immediately after running "source ${HELPER}". The
blob_fixup() function will be called by the common extract() function
after extracting every individual blob, giving the user the
opportunity to hook custom code after this operation takes place.
* In proprietary-files.txt, the line corresponding to this blob which
needs fixups can look in one of 2 ways:
(a) vendor/lib64/vendor.qti.gnss@1.0_vendor.so
Do this if you are taking the blob from the stock ROM. The fixup
script will always run after the blob is extracted.
(b) vendor/lib64/vendor.qti.gnss@1.0_vendor.so|249c76153f8de014bf2dd2ab623ee3d87741fbc8|f7e9ee8e3804887a2f3939128e860767e6f27258
Do this if you are kanging the blob from somebody else. The pinning
logic now applies for both the pre- and the post-fixup hashes. The
fixup script will only run if the blob doesn't match the hex-edited blob,
although the fixup script should really be idempotent.
Change-Id: Ifdd73c885d995c645f6210597537693d1a2f903f
Signed-off-by: Vladimir Oltean <olteanv@gmail.com>
diff --git a/build/tools/extract_utils.sh b/build/tools/extract_utils.sh
index 143ff2e..fb39fa9 100644
--- a/build/tools/extract_utils.sh
+++ b/build/tools/extract_utils.sh
@@ -17,8 +17,10 @@
PRODUCT_COPY_FILES_LIST=()
PRODUCT_COPY_FILES_HASHES=()
+PRODUCT_COPY_FILES_FIXUP_HASHES=()
PRODUCT_PACKAGES_LIST=()
PRODUCT_PACKAGES_HASHES=()
+PRODUCT_PACKAGES_FIXUP_HASHES=()
PACKAGE_LIST=()
VENDOR_STATE=-1
VENDOR_RADIO_STATE=-1
@@ -674,8 +676,10 @@
PRODUCT_PACKAGES_LIST=()
PRODUCT_PACKAGES_HASHES=()
+ PRODUCT_PACKAGES_FIXUP_HASHES=()
PRODUCT_COPY_FILES_LIST=()
PRODUCT_COPY_FILES_HASHES=()
+ PRODUCT_COPY_FILES_FIXUP_HASHES=()
while read -r line; do
if [ -z "$line" ]; then continue; fi
@@ -687,17 +691,23 @@
local COUNT=${#SPLIT[@]}
local SPEC=${SPLIT[0]}
local HASH="x"
+ local FIXUP_HASH="x"
if [ "$COUNT" -gt "1" ]; then
HASH=${SPLIT[1]}
fi
+ if [ "$COUNT" -gt "2" ]; then
+ FIXUP_HASH=${SPLIT[2]}
+ fi
# if line starts with a dash, it needs to be packaged
if [[ "$SPEC" =~ ^- ]]; then
PRODUCT_PACKAGES_LIST+=("${SPEC#-}")
PRODUCT_PACKAGES_HASHES+=("$HASH")
+ PRODUCT_PACKAGES_FIXUP_HASHES+=("$FIXUP_HASH")
else
PRODUCT_COPY_FILES_LIST+=("$SPEC")
PRODUCT_COPY_FILES_HASHES+=("$HASH")
+ PRODUCT_COPY_FILES_FIXUP_HASHES+=("$FIXUP_HASH")
fi
done < <(egrep -v '(^#|^[[:space:]]*$)' "$LIST" | LC_ALL=C sort | uniq)
@@ -917,12 +927,23 @@
mv "$TEMP_XML" "$XML"
}
+function get_hash() {
+ local FILE="$1"
+
+ if [ "$(uname)" == "Darwin" ]; then
+ shasum "${FILE}" | awk '{print $1}'
+ else
+ sha1sum "${FILE}" | awk '{print $1}'
+ fi
+}
+
function print_spec() {
local SPEC_PRODUCT_PACKAGE="$1"
local SPEC_SRC_FILE="$2"
local SPEC_DST_FILE="$3"
local SPEC_ARGS="$4"
local SPEC_HASH="$5"
+ local SPEC_FIXUP_HASH="$6"
local PRODUCT_PACKAGE=""
if [ ${SPEC_PRODUCT_PACKAGE} = true ]; then
@@ -944,7 +965,22 @@
if [ ! -z "${SPEC_HASH}" ] && [ "${SPEC_HASH}" != "x" ]; then
HASH="|${SPEC_HASH}"
fi
- printf '%s%s%s%s%s\n' "${PRODUCT_PACKAGE}" "${SRC}" "${DST}" "${ARGS}" "${HASH}"
+ local FIXUP_HASH=""
+ if [ ! -z "${SPEC_FIXUP_HASH}" ] && [ "${SPEC_FIXUP_HASH}" != "x" ] && [ "${SPEC_FIXUP_HASH}" != "${SPEC_HASH}" ]; then
+ FIXUP_HASH="|${SPEC_FIXUP_HASH}"
+ fi
+ printf '%s%s%s%s%s%s\n' "${PRODUCT_PACKAGE}" "${SRC}" "${DST}" "${ARGS}" "${HASH}" "${FIXUP_HASH}"
+}
+
+# To be overridden by device-level extract-files.sh
+# Parameters:
+# $1: spec name of a blob. Can be used for filtering.
+# If the spec is "src:dest", then $1 is "dest".
+# If the spec is "src", then $1 is "src".
+# $2: path to blob file. Can be used for fixups.
+#
+function blob_fixup() {
+ :
}
#
@@ -1004,6 +1040,7 @@
local FILELIST=( ${PRODUCT_COPY_FILES_LIST[@]} ${PRODUCT_PACKAGES_LIST[@]} )
local HASHLIST=( ${PRODUCT_COPY_FILES_HASHES[@]} ${PRODUCT_PACKAGES_HASHES[@]} )
+ local FIXUP_HASHLIST=( ${PRODUCT_COPY_FILES_FIXUP_HASHES[@]} ${PRODUCT_PACKAGES_FIXUP_HASHES[@]} )
local PRODUCT_COPY_FILES_COUNT=${#PRODUCT_COPY_FILES_LIST[@]}
local COUNT=${#FILELIST[@]}
local OUTPUT_ROOT="$LINEAGE_ROOT"/"$OUTDIR"/proprietary
@@ -1096,20 +1133,17 @@
# Check pinned files
local HASH="$(echo ${HASHLIST[$i-1]} | awk '{ print tolower($0); }')"
+ local FIXUP_HASH="$(echo ${FIXUP_HASHLIST[$i-1]} | awk '{ print tolower($0); }')"
local KEEP=""
- if [ "$DISABLE_PINNING" != "1" ] && [ ! -z "$HASH" ] && [ "$HASH" != "x" ]; then
+ if [ "$DISABLE_PINNING" != "1" ] && [ "$HASH" != "x" ]; then
if [ -f "${VENDOR_REPO_FILE}" ]; then
local PINNED="${VENDOR_REPO_FILE}"
else
local PINNED="${TMP_DIR}${DST_FILE#/system}"
fi
if [ -f "$PINNED" ]; then
- if [ "$(uname)" == "Darwin" ]; then
- local TMP_HASH=$(shasum "$PINNED" | awk '{print $1}' )
- else
- local TMP_HASH=$(sha1sum "$PINNED" | awk '{print $1}' )
- fi
- if [ "$TMP_HASH" = "$HASH" ]; then
+ local TMP_HASH=$(get_hash "${PINNED}")
+ if [ "${TMP_HASH}" = "${HASH}" ] || [ "${TMP_HASH}" = "${FIXUP_HASH}" ]; then
KEEP="1"
if [ ! -f "${VENDOR_REPO_FILE}" ]; then
cp -p "$PINNED" "${VENDOR_REPO_FILE}"
@@ -1143,19 +1177,23 @@
fi
fi
- if [ "$?" == "0" ]; then
- # Deodex apk|jar if that's the case
- if [[ "$FULLY_DEODEXED" -ne "1" && "${VENDOR_REPO_FILE}" =~ .(apk|jar)$ ]]; then
- oat2dex "${VENDOR_REPO_FILE}" "${SRC_FILE}" "$SRC"
- if [ -f "$TMPDIR/classes.dex" ]; then
- zip -gjq "${VENDOR_REPO_FILE}" "$TMPDIR/classes.dex"
- rm "$TMPDIR/classes.dex"
- printf ' (updated %s from odex files)\n' "${SRC_FILE}"
- fi
- elif [[ "${VENDOR_REPO_FILE}" =~ .xml$ ]]; then
- fix_xml "${VENDOR_REPO_FILE}"
+ # Blob fixup pipeline has 2 parts: one that is fixed and
+ # one that is user-configurable
+ local PRE_FIXUP_HASH=$(get_hash ${VENDOR_REPO_FILE})
+ # Deodex apk|jar if that's the case
+ if [[ "$FULLY_DEODEXED" -ne "1" && "${VENDOR_REPO_FILE}" =~ .(apk|jar)$ ]]; then
+ oat2dex "${VENDOR_REPO_FILE}" "${SRC_FILE}" "$SRC"
+ if [ -f "$TMPDIR/classes.dex" ]; then
+ zip -gjq "${VENDOR_REPO_FILE}" "$TMPDIR/classes.dex"
+ rm "$TMPDIR/classes.dex"
+ printf ' (updated %s from odex files)\n' "${SRC_FILE}"
fi
+ elif [[ "${VENDOR_REPO_FILE}" =~ .xml$ ]]; then
+ fix_xml "${VENDOR_REPO_FILE}"
fi
+ # Now run user-supplied fixup function
+ blob_fixup "${BLOB_DISPLAY_NAME}" "${VENDOR_REPO_FILE}"
+ local POST_FIXUP_HASH=$(get_hash ${VENDOR_REPO_FILE})
if [ -f "${VENDOR_REPO_FILE}" ]; then
local DIR=$(dirname "${VENDOR_REPO_FILE}")
@@ -1168,7 +1206,19 @@
fi
if [ "${KANG}" = true ]; then
- print_spec "${IS_PRODUCT_PACKAGE}" "${SPEC_SRC_FILE}" "${SPEC_DST_FILE}" "${SPEC_ARGS}" "${HASH}"
+ print_spec "${IS_PRODUCT_PACKAGE}" "${SPEC_SRC_FILE}" "${SPEC_DST_FILE}" "${SPEC_ARGS}" "${PRE_FIXUP_HASH}" "${POST_FIXUP_HASH}"
+ fi
+
+ # Check and print whether the fixup pipeline actually did anything.
+ # This isn't done right after the fixup pipeline because we want this print
+ # to come after print_spec above, when in kang mode.
+ if [ "${PRE_FIXUP_HASH}" != "${POST_FIXUP_HASH}" ]; then
+ printf " + Fixed up %s\n" "${BLOB_DISPLAY_NAME}"
+ # Now sanity-check the spec for this blob.
+ if [ "${KANG}" = false ] && [ "${FIXUP_HASH}" = "x" ] && [ "${HASH}" != "x" ]; then
+ printf "WARNING: The %s file was fixed up, but it is pinned.\n" ${BLOB_DISPLAY_NAME}
+ printf "This is a mistake and you want to either remove the hash completely, or add an extra one.\n"
+ fi
fi
done