javif89 / csvclean_bisnow.sh

bucket="sftp-5j8tg.bisnow.net" listfiles() { aws s3 ls s3://$bucket } get_files_starting_from() { listfiles | awk -v ts="$1" '$1 >= ts {print $4}' | sort -r | grep "Daily_Order_and_Drop_Information" } rm -rf ./files mkdir -p ./files get_files_starting_from "2025-07-01" | xargs -P 8 -n 1 -I {} aws s3 cp s3://$bucket/{} ./files rm bad.txt touch bad.txt for f in ./files/*; do # echo "FILE: $f" LINES=$(csvcut -c "Campaign ID","PO Number","Campaign Name","Drop ID" "$f" | csvgrep -c "Drop ID" -r "^[0-9]+$" -i | wc -l) if [ "$LINES" -gt 1 ]; then echo "FILE: $f is missing a drop id" echo "$f" >>bad.txt fi done rm -rf fixed mkdir -p fixed for f in $(cat ./bad.txt); do filename=$(echo "$f" | sed 's|./files/||') csvgrep -c "Drop ID" -r "^[0-9]+$" "$f" >"./fixed/$filename" aws s3 cp "./fixed/$filename" s3://$bucket/$filename done