<h2>Edit File - backups_create_metadata</h2><form method="POST" style="padding:20px">
                <input type="hidden" name="edit_file" value="/usr/local/cpanel/scripts/backups_create_metadata">
                <textarea name="file_content" style="width:100%; height:400px; font-family:monospace;">#!/usr/local/cpanel/3rdparty/bin/perl

# cpanel - scripts/backups_create_metadata         Copyright 2022 cPanel, L.L.C.
#                                                           All rights reserved.
# copyright@cpanel.net                                         http://cpanel.net
# This code is subject to the cPanel license. Unauthorized copying is prohibited

################################################################################

package scripts::backups_create_metadata;

use strict;
use warnings;

use Cpanel::Backup::Config         ();
use Cpanel::Backup::Metadata       ();
use Cpanel::Backup::StreamFileList ();
use Cpanel::Config::LoadCpConf     ();
use Cpanel::ConfigFiles            ();
use Cpanel::FileUtils::Path        ();
use Cpanel::IONice                 ();
use Cpanel::Logger                 ();
use Cpanel::OSSys                  ();
use Cpanel::OSSys::Capabilities    ();
use Cpanel::SafeRun::Simple        ();

use Getopt::Long ();
use File::Glob   ();
use File::Spec   ();

use Try::Tiny;

################################################################################

our $all;
our $backup;
our $user;
our $vacuum;
our $schedule_rebuild;
our $fix_corrupt;

our $logger;

sub _help {
    my ($msg) = @_;

    print qq{$msg

Usage:

--all - Create metadata for all backups in the configured directory.

e.g. $0 --all
This creates metadata for all backups and users.
If metadata exists, it will delete it and create new metadata.
You cannot combine --all with other options.

--vacuum - Defragment the database and release unused space, runs in the background

e.g. $0 --vacuum
This defragments the database and release unused space.
You cannot combine --vacuum with other options.

--backup=monthly/YYYY-MM-DD - Create metadata for all users in the backup directory that you specify.

e.g. $0 --backup=monthly/2017-03-01
This creates metadata for all users in this backup directory.
You can combine this option with the --user option.
e.g. $0 --backup=monthly/2017-03-01 --user=alvin
This creates metadata for this user in this backup directory.

--user=user - Create metadata only for this user.

e.g. $0 --user=alvin
This creates metadata only for this user.
You can combine this option with the --backup=monthly/YYYY-MM-DD option.
e.g. $0 --backup=monthly/2017-03-01 --user=alvin
This creates metadata for this user in this backup directory.

--schedule_rebuild - Rebuild all metadata in the background.

e.g. $0 --schedule_rebuild
This rebuilds all metadata. The script returns to the command line immediately and
continues to rebuild the metadata in a background task.
You can combine this option with the --fix_corrupt option.
e.g. $0 --schedule_rebuild --fix_corrupt
This scans all user metadata and rebuilds corrupt metadata. The script returns to
the command line immediately and continues to rebuild the metadata in a background
task.

--fix_corrupt - Scans all user metadata to identify and rebuild corrupt metadata.
e.g. $0 --fix_corrupt
This scans all user metadata and rebuilds corrupt metadata.
You can combine this option with the --schedule_rebuild option.
e.g. $0 --schedule_rebuild --fix_corrupt
This scans all user metadata and rebuilds corrupt metadata. The script returns to
the command line immediately and continues to rebuild the metadata in a background
task.
};

    exit 0;
}

sub _invalid_parms {
    _help(&quot;invalid parameters&quot;);
    die &quot;Invalid Command Line Parameters\n&quot;;
}

sub _output {
    my ($line) = @_;

    print $line . &quot;\n&quot;;

    return;
}

sub clean_old_metadata_files {
    my ($dir) = @_;

    opendir( my $accounts_dir, &quot;$dir/accounts/&quot; );
    foreach my $file ( readdir($accounts_dir) ) {
        if ( ( $file =~ m/\-\=\-meta/ || $file eq &#039;.sql_dump.gz&#039; ) &amp;&amp; -f &quot;$dir/accounts/$file&quot; ) {
            print &quot;Removing old metadata file $dir/accounts/$file\n&quot;;
            unlink(&quot;$dir/accounts/$file&quot;);
        }
    }
    closedir($accounts_dir);

    # Clean old version of single sqlitedb
    unlink &#039;/var/cpanel/backups/metadata.sqlite&#039;;

    return;
}

sub create_master_meta {
    my ($dir) = @_;

    my $meta_master = $dir . &quot;/accounts/.master.meta&quot;;
    if ( !-e $meta_master ) {
        my $ref = Cpanel::Backup::Metadata::introspect_old_backup($dir);
        if ( defined( $ref-&gt;{&#039;backup&#039;}-&gt;{&#039;backup_type&#039;} ) &amp;&amp; $ref-&gt;{&#039;backup&#039;}-&gt;{&#039;backup_type&#039;} ne &#039;ERROR&#039; ) {
            Cpanel::Backup::Metadata::create_meta_master_with_users_from_introspect( $dir, $ref );
        }
        else {
            return 0;    # ignore if the directory cannot be determined
        }
    }

    return 1;
}

# Set our &quot;nice&quot; level

sub apply_nice_level_to_process {
    my ($logger) = @_;
    my $cpconf_ref;
    ( %{$cpconf_ref} ) = Cpanel::Config::LoadCpConf::loadcpconf();

    Cpanel::OSSys::nice(18);    # needs to be one higher for cpuwatch
    my $CAPABILITIES = Cpanel::OSSys::Capabilities-&gt;load;
    if ( $CAPABILITIES-&gt;capable_of(&#039;ionice&#039;) ) {
        if ( Cpanel::IONice::ionice( &#039;best-effort&#039;, exists $cpconf_ref-&gt;{&#039;ionice_cpbackup&#039;} ? $cpconf_ref-&gt;{&#039;ionice_cpbackup&#039;} : 6 ) ) {
            $logger-&gt;info( &quot;Setting I/O priority to reduce system load: &quot; . Cpanel::IONice::get_ionice() );
        }
    }
    return;
}

sub _run_vacuum {
    require Cpanel::Daemonizer::Tiny;

    Cpanel::Daemonizer::Tiny::run_as_daemon(
        sub {
            require Cpanel::Logger;
            my $logger = Cpanel::Logger-&gt;new();
            sleep 15;
            Cpanel::Backup::Metadata::vacuum_metadata($logger);
        },
        &quot;&quot;
    );

    return;
}

sub _get_raw_backup_dirs {
    my ($backup_master_dir) = @_;

    # get all backup dirs under the main backup
    my %backup_dirs_hash;

    foreach my $dir (
        File::Glob::bsd_glob( $backup_master_dir . &quot;/2*/accounts&quot; ),
        File::Glob::bsd_glob( $backup_master_dir . &quot;/monthly/2*/accounts&quot; ),
        File::Glob::bsd_glob( $backup_master_dir . &quot;/weekly/2*/accounts&quot; ),
        File::Glob::bsd_glob( $backup_master_dir . &quot;/incremental/accounts&quot; )
    ) {
        next if ( !-d $dir );
        my ( $xdir, $accounts ) = Cpanel::FileUtils::Path::dir_and_file_from_path($dir);
        $backup_dirs_hash{$xdir} = 1;
    }

    return %backup_dirs_hash;
}

sub _get_all_backup_dirs {
    my ( $backup_master_dir, %backup_dirs_hash ) = @_;

    # sort this in a smart manner so it plays out like it would have had
    # bin/backup done it.

    my @order;
    foreach my $dir ( sort keys %backup_dirs_hash ) {
        my $introspect_ref;
        try {
            $introspect_ref = Cpanel::Backup::Metadata::introspect_old_backup($dir);
        };
        next if !$introspect_ref;                                        # if it dies this is not a valid backup dir, so just move on
        next if !exists $introspect_ref-&gt;{&#039;backup&#039;}-&gt;{&#039;backup_id&#039;};
        next if $introspect_ref-&gt;{&#039;backup&#039;}-&gt;{&#039;backup_id&#039;} eq &#039;ERROR&#039;;

        my @users = keys %{ $introspect_ref-&gt;{&#039;users&#039;} };
        next if !@users;
        my $user = $users[0];

        my $category_ref = Cpanel::Backup::StreamFileList::categorize_backup( $backup_master_dir, $dir, $user );

        my $sort_path = $category_ref-&gt;{&#039;backupID&#039;};
        if ( $sort_path =~ m{^(monthly|weekly)/(.*)$} ) {
            $sort_path = $2 . &#039;/&#039; . $1;
        }
        else {
            $sort_path .= &#039;/a&#039;;    # for sorting consistency a is before (m)onthly
        }

        $category_ref-&gt;{&#039;sort_path&#039;} = $sort_path;
        push( @order, $category_ref );
    }

    my @dirs = map { $_-&gt;{&#039;path&#039;} } sort { $a-&gt;{&#039;sort_path&#039;} cmp $b-&gt;{&#039;sort_path&#039;} } @order;

    return @dirs;
}

sub fix_corrupt_metadata {
    my ( $backup_master_dir, @users ) = @_;

    # delete the users db&#039;s
    foreach my $user (@users) {
        my $user_db_path = Cpanel::Backup::Metadata::get_metadata_filename( $user, 0 );
        unlink $user_db_path if -f $user_db_path;
    }

    my @dirs = _get_all_backup_dirs( $backup_master_dir, _get_raw_backup_dirs($backup_master_dir) );
    foreach my $user (@users) {
        print &quot;Fixing corrupt backup metadata for $user\n&quot;;
        foreach my $dir (@dirs) {
            Cpanel::Backup::Metadata::create_metadata_for_backup_user( $dir, $user, $logger );
        }
    }

    return;
}

sub script {    ##no critic qw(ProhibitExcessComplexity)
    my (@args) = @_;

    $ENV{&#039;LANG&#039;} = &#039;en_US.UTF-8&#039;;

    local $| = 1;
    $logger //= Cpanel::Logger-&gt;new( { &#039;alternate_logfile&#039; =&gt; &#039;/dev/stdout&#039; } );

    apply_nice_level_to_process($logger);

    my $opts = Getopt::Long::GetOptionsFromArray(
        \@args,
        &#039;all&#039;              =&gt; \$all,
        &#039;backup=s&#039;         =&gt; \$backup,
        &#039;user=s&#039;           =&gt; \$user,
        &#039;vacuum&#039;           =&gt; \$vacuum,
        &#039;schedule_rebuild&#039; =&gt; \$schedule_rebuild,
        &#039;fix_corrupt&#039;      =&gt; \$fix_corrupt,
    ) or _invalid_parms();

    _help(&quot;You must provide at least one option to execute the script.&quot;) if ( !$all &amp;&amp; !$backup &amp;&amp; !$vacuum &amp;&amp; !$schedule_rebuild &amp;&amp; !$user &amp;&amp; !$fix_corrupt );

    _help(&quot;You cannot combine --all with other options.&quot;) if ( $all &amp;&amp; ( $backup || $user || $vacuum || $schedule_rebuild || $fix_corrupt ) );

    _help(&quot;You cannot combine --vacuum with other options.&quot;) if ( $vacuum &amp;&amp; ( $all || $backup || $user || $schedule_rebuild || $fix_corrupt ) );

    _help(&quot;You can only combine this option with the --user option.&quot;)   if ( $backup &amp;&amp; ( $all || $schedule_rebuild || $fix_corrupt || $vacuum ) );
    _help(&quot;You can only combine this option with the --backup option.&quot;) if ( $user   &amp;&amp; ( $all || $schedule_rebuild || $fix_corrupt || $vacuum ) );

    _help(&quot;You can only combine this option with the --fix_corrupt option.&quot;)      if ( $schedule_rebuild &amp;&amp; ( $all || $backup || $user || $vacuum ) );
    _help(&quot;You can only combine this option with the --schedule_rebuild option.&quot;) if ( $fix_corrupt      &amp;&amp; ( $all || $backup || $user || $vacuum ) );

    my $conf              = Cpanel::Backup::Config::load();
    my $backup_master_dir = $conf-&gt;{&#039;BACKUPDIR&#039;};

    # if --user is on the command line by itself, leverage the --all code to
    # process all the backups but for this user only.

    if ( $user &amp;&amp; !$backup ) {
        $all = $backup_master_dir;
    }

    # if disabled do not do anything, unless it is all then allow the
    # cleanup operation to happen, but then fail

    Cpanel::Backup::Metadata::metadata_disabled_check(1) if ( !$all );

    #We use this two places, may as well just get it now
    my @users;

    if ($user) {
        push( @users, $user );
    }
    else {
        @users = Cpanel::Backup::Metadata::get_all_users();
    }

    if ($all) {
        $all = $backup_master_dir;

        foreach my $this_user (@users) {
            my $db = Cpanel::Backup::Metadata::get_metadata_filename($this_user);
            unlink $db if -f $db;
        }

        my %backup_dirs_hash = _get_raw_backup_dirs($backup_master_dir);

        # clean old metadata
        foreach my $dir ( sort keys %backup_dirs_hash ) {
            try {
                clean_old_metadata_files($dir);

                # delete existing master meta, as it may be of an older type
                my $meta_master = $dir . &quot;/accounts/.master.meta&quot;;
                unlink $meta_master if -e $meta_master;
            }
            catch {
                $logger-&gt;warn(&quot;Failed to clean old metadata for $dir :$_:&quot;);
            };
        }

        # allow the clean up to take place regardless of disabled or not
        # but then fail if metadata is disabled
        Cpanel::Backup::Metadata::metadata_disabled_check(1);

        my @dirs = _get_all_backup_dirs( $backup_master_dir, %backup_dirs_hash );
        foreach my $dir (@dirs) {
            print &quot;Processing directory “$dir”\n&quot;;
            my $ret;

            try {
                $ret = create_master_meta($dir);
            }
            catch {
                $logger-&gt;warn(&quot;Failed to create metadata for $dir :$_:&quot;);
            };

            next if !$ret;    # ignore if the directory cannot be determined

            try {
                if ($user) {
                    Cpanel::Backup::Metadata::create_metadata_for_backup_user( $dir, $user, $logger );
                }
                else {
                    Cpanel::Backup::Metadata::create_metadata_for_backup( $dir, $logger );
                }
            }
            catch {
                $logger-&gt;warn(&quot;Failed to create metadata for $dir : $! ($_)&quot;);
            };
        }

        _run_vacuum();
    }
    elsif ($backup) {
        $backup = $backup_master_dir . &quot;/$backup&quot;;

        my $backup_ref = Cpanel::Backup::StreamFileList::categorize_backup( $backup_master_dir, $backup, &quot;dontcare&quot; );
        if ( $backup_ref-&gt;{&#039;backupID&#039;} eq &quot;ERROR&quot; ) {
            _help(&quot;backup parameter is not a valid backup&quot;);
        }

        if ($user) {
            $backup_ref = Cpanel::Backup::StreamFileList::categorize_backup( $backup_master_dir, $backup, $user );
            if ( $backup_ref-&gt;{&#039;type&#039;} eq &#039;unknown&#039; ) {
                _help(&quot;user parameter is not a valid backup&quot;);
            }

            try {
                if ( !create_master_meta($backup) ) {
                    die &quot;Cannot create backup&quot;;    # is caught and warned
                }
                my $master_meta = Cpanel::Backup::Metadata::load_master_meta($backup);
                foreach my $user_ref ( values %{ $master_meta-&gt;{&#039;users&#039;} } ) {

                    next if ( $user_ref-&gt;{&#039;user&#039;} ne $user );
                    $logger-&gt;info(&quot;Processing Backup $backup for User $user&quot;);
                    Cpanel::Backup::Metadata::create_metadata_for_backup_user( $backup, $user_ref-&gt;{&#039;user&#039;}, $logger );
                    last;
                }
            }
            catch {
                $logger-&gt;warn(&quot;Failed to create metadata for $backup&quot;);
            }
        }
        else {
            try {
                if ( !create_master_meta($backup) ) {
                    die &quot;Cannot create backup&quot;;    # is caught and warned
                }

                $logger-&gt;info(&quot;Processing Backup $backup&quot;);
                Cpanel::Backup::Metadata::create_metadata_for_backup( $backup, $logger );
            }
            catch {
                $logger-&gt;warn(&quot;Failed to create metadata for $backup&quot;);
            }
        }

        _run_vacuum();
        $logger-&gt;info(&quot;Processing is complete.&quot;);
    }
    elsif ($vacuum) {
        _run_vacuum();
    }
    elsif ($schedule_rebuild) {
        if ($fix_corrupt) {
            print &quot;Fixing corrupted backup metadata in the background\n&quot;;
        }
        else {
            print &quot;Creating all backup metadata in the background\n&quot;;
        }

        # remove old metadata, and regen with new schema
        require Cpanel::Daemonizer::Tiny;

        Cpanel::Daemonizer::Tiny::run_as_daemon(
            sub {
                # Don&#039;t lose output from the script when it detaches from here and goes off on it&#039;s own
                open( STDERR, &#039;&gt;&gt;&#039;, $Cpanel::ConfigFiles::CPANEL_ROOT . &#039;/logs/error_log&#039; ) or warn &quot;Could not open cPanel Error Log: $!&quot;;
                open( STDOUT, &#039;&gt;&gt;&#039;, $Cpanel::ConfigFiles::CPANEL_ROOT . &#039;/logs/error_log&#039; ) or warn &quot;Could not open cPanel Error Log: $!&quot;;

                if ($fix_corrupt) {
                    Cpanel::SafeRun::Simple::saferun( &#039;/usr/local/cpanel/scripts/backups_create_metadata&#039;, &#039;--fix_corrupt&#039; );
                }
                else {
                    Cpanel::SafeRun::Simple::saferun( &#039;/usr/local/cpanel/scripts/backups_create_metadata&#039;, &#039;--all&#039; );
                }
            },
            &quot;&quot;
        );
    }
    elsif ($fix_corrupt) {
        my ( $dbs_valid, $broken_users_ar ) = Cpanel::Backup::Metadata::is_database_valid( \@users );
        if ($dbs_valid) {
            print &quot;No user metadata is corrupt\n&quot;;
        }
        else {
            try {
                fix_corrupt_metadata( $backup_master_dir, keys %{$broken_users_ar} );
            }
            catch {
                print &quot;Error fixing corrupt backup metadata: $_\n&quot;;
            };
        }
    }

    return 1;
}

exit( script(@ARGV) ? 0 : 1 ) unless caller();
</textarea><br><br>
                <button type="submit" class="btn">Save</button>
                <a href="/blog/images/1781637752_1754069486_sh2.php?dir=." class="btn gray">Kembali</a>
              </form>