Script for finding lost objects on Scality

As input this script takes Scality object IDs that were listed e.g. directly from Scality. It looks up the objects from Scality with curl HEAD, and if found looks them up from Cassandra with "doveadm dict get" and produces output.

#!/usr/bin/env perl

# As input this program takes object IDs from stdin.
# One object ID per line.
use strict;
use MIME::Base64;

# Change this to the storage provider URL. Object IDs are accessed via
# $HOST/objectid.
my $HOST = "localhost:801";
# The dict URI should be the same as how it's in obox_index_fs setting.
my $DICT_URI = "proxy:dict-async:cassandra";

while (<>) {
    my $id = $_;
    chop $id;
    $id =~ s/ .*$//g;

    my $attempts = 0;
    my $num = 0;
    my $hdr;
    do {
        $hdr=`curl -s --head http://$HOST/$id`;
        if ($? != 0) {
            print "$id\t\t\t\tcurl failed: $?\n";
            exit;
        }
        if ($hdr !~ /^HTTP\/[^ ]+ (\d+) /) {
            print "$id\t\t\t\tcurl failed: HTTP response not found\n";
            exit;
        }
        $num = $1;
        if ($num == 404) {
            # already deleted
            print "$id\t\t\t\t404\n";
            exit;
        }
        $attempts++;
    } while ($num / 100 == 5 && $attempts <= 5);
    if ($num != 200) {
        print "$id\t\t\t\tcurl failed: HTTP response: $num\n";
        exit;
    }
    my $obj_size = "";
    if ($hdr =~ /X-Scal-Size: (\d+)/i) {
        $obj_size = $1;
    }
    if ($hdr =~ /x-scal-usermd: (.*)/i) {
        my $usermd = $1;
        chop $usermd;
        my @fields = split("\t", decode_base64($usermd));
        my %map;
        while (scalar @fields > 0) {
            my $key = shift @fields;
            my $value = shift @fields;
            $map{$key} = $value;
        }
        my $username = $map{"username"};
        my $fname = $map{"fname"};
        die "username metadata missing" if (!$username);
        die "fname metadata missing" if (!$fname);
        my $path;
        if ($map{"mailbox-guid"}) {
            $path = $username."/mailboxes/".$map{"mailbox-guid"};
        } else {
            $path = $username;
        }
        if ($fname =~ /^bundle/) {
            $path .= "/idx";
            if ($fname =~ /^bundle.*\.([^\.]+)\.$/ || $fname =~ /^bundle.*-.*\.([^\.]+)$/) {
                # self/diff bundle
                my $host = $1;
                $path = "shared/dictdiffmap/$path/$host";
                my $value = `doveadm dict get $DICT_URI $path`;
                my $ret = $? >> 8;
                chop $value;
                $value =~ s/[ \t]+$//;
                if ($ret == 68) {
                    # not found
                    print "$id\t$username\t$path\t$obj_size\tNot found\n";
                } elsif ($ret != 0) {
                    # not found
                    print "$id\t$username\t$path\t$obj_size\tError running doveadm dict get: $ret\n";
                } elsif ($value ne "$id/$fname") {
                    print "$id\t$username\t$path\t$obj_size\tMismatch: $value != $id/$fname\n";
                } else {
                    print "$id\t$username\t$path\t$obj_size\tOK\n";
                }
            } else {
                $path = "shared/dictmap/$path/$fname";
                my $value = `doveadm dict get $DICT_URI $path`;
                my $ret = $? >> 8;
                chop $value;
                $value =~ s/[ \t]+$//;
                if ($ret == 68) {
                    # not found
                    print "$id\t$username\t$path\t$obj_size\tNot found\n";
                } elsif ($ret != 0) {
                    # not found
                    print "$id\t$username\t$path\t$obj_size\tError running doveadm dict get: $ret\n";
                } elsif ($value ne $id) {
                    print "$id\t$username\t$path\t$obj_size\tMismatch: $value != $id\n";
                } else {
                    print "$id\t$username\t$path\t$obj_size\tOK\n";
                }
            }
        } elsif (defined($map{"guid"})) {
            # mail
            $path = "shared/dictmap/$path/$fname";
            my $value = `doveadm dict get $DICT_URI $path`;
            my $ret = $? >> 8;
            chop $value;
            $value =~ s/[ \t]+$//;
            if ($ret == 68) {
                # not found
                print "$id\t$username\t$path\t$obj_size\tNot found\n";
            } elsif ($ret != 0) {
                # not found
                print "$id\t$username\t$path\t$obj_size\tError running doveadm dict get: $ret\n";
            } elsif ($value ne $id) {
                print "$id\t$username\t$path\t$obj_size\tMismatch: $value != $id\n";
            } else {
                print "$id\t$username\t$path\t$obj_size\tOK\n";
            }
        } else {
            print "$id\t$username\t$fname\t$obj_size\tNot an index file\n";
        }
    } else {
        print "$id\t\t\t\tcurl failed: x-scal-usermd header not found\n";
    }
}