Mini Shell
#!/opt/imh-python/bin/python3
import arrow
import shutil
import csv
import os
import argparse
import base64
import requests
from rads.common import colors
col = colors()
import ftplib
import socket
import picker
from rads.common import strcolor
import string
import random
from rads.cpanel_api import cpanel_api
from rads.common import is_cpanel_user
import sys
import time
from glob import glob
from prettytable import PrettyTable
from rads.common import yes_or_no
def welcome(parsed, maninput):
print(
"We will attempt to transfer all cPanel users from the reseller {} at the host {}".format(
parsed.User, parsed.RemoteHost
)
)
if maninput:
print('If you need to run this again you can use the shortened version')
print(
"./remote_dump --user {} --pass '{}' --host {} --ftphost {} --ftpuser {} --ftppass '{}'".format(
parsed.User,
parsed.ResPass,
parsed.RemoteHost,
parsed.FTPhost,
parsed.FTPuser,
parsed.FTPpass,
)
)
if not parsed.use_ssl:
print(
strcolor(
'red',
'If you have troubles, you might want to try the --ssl flag',
)
)
def update_csv(user, status, csvfile):
fo_append = ''.join(
random.choice(
string.ascii_uppercase + string.digits + string.ascii_lowercase
)
for _ in range(6)
)
f = open(csvfile, 'rb')
fo_file = f'/tmp/status-{fo_append}.csv'
fo = open(fo_file, 'wb')
# go through each line of the file
for line in f:
bits = line.split(',')
# change second column
if bits[0] == user:
bits[1] = status
# join it back together and write it out
fo.write(','.join(bits))
f.close()
fo.close()
shutil.copyfile(fo_file, csvfile)
def success_from_path(path):
pending = 0
transfer = 0
backup_file = {}
backups = glob(f'{path}/backup-*.tar.gz')
if os.path.isfile(f'{path}/status.csv'):
with open(f'{path}/status.csv', 'rb') as csvfile:
csvinfo = csv.DictReader(csvfile, delimiter=',')
for row in csvinfo:
for file in backups:
if file.endswith('_{}.tar.gz'.format(row['user'])):
row['status'] = 'Transferring'
backupfile = file
backup_file[row['user']] = file
if row['status'] == 'Transferring':
with open('/var/log/messages') as logfile:
for line in logfile:
if (
'{} uploaded'.format(backupfile.split('/')[-1])
in line
):
row['status'] = 'Complete'
if row['status'] == 'Complete' and not os.path.isfile(
'/var/cpanel/users/{}'.format(row['user'])
):
print(backup_file[row['user']])
def status_from_path(path, timeout):
pending = 0
transfer = 0
backup_file = {}
backups = glob(f'{path}/backup-*.tar.gz')
if os.path.isfile(f'{path}/status.csv'):
print('Found Status File')
with open(f'{path}/status.csv', 'rb') as csvfile:
csvinfo = csv.DictReader(csvfile, delimiter=',')
for row in csvinfo:
for file in backups:
if file.endswith('_{}.tar.gz'.format(row['user'])):
if row['status'] != 'Complete':
row['status'] = 'Transferring'
backupfile = file
backup_file[row['user']] = file
if row['status'] == 'Transferring':
with open('/var/log/messages') as logfile:
for line in logfile:
if (
'{} uploaded'.format(backupfile.split('/')[-1])
in line
):
row['status'] = 'Complete'
update_csv(
row['user'],
row['status'],
f'{path}/status.csv',
)
if os.path.isfile(
'{}/{}.started'.format(path, row['user'])
):
os.remove(
'{}/{}.started'.format(
path, row['user']
)
)
if row['status'] != 'Waiting' and row['status'] != 'Pending':
update_csv(row['user'], row['status'], f'{path}/status.csv')
if row['status'] == 'Pending':
if os.path.isfile(
'{}/{}.started'.format(path, row['user'])
):
print(
'{} Start time was {} seconds ago'.format(
row['user'],
time.time()
- os.path.getctime(
'{}/{}.started'.format(path, row['user'])
),
)
)
if (
time.time()
- os.path.getctime(
'{}/{}.started'.format(path, row['user'])
)
> timeout
):
print('{} Timed Out'.format(row['user']))
update_csv(
row['user'], 'Timed Out', f'{path}/status.csv'
)
row['status'] == 'Timed Out'
if row['status'] == 'Pending':
pending += 1
if row['status'] == 'Transferring':
transfer += 1
x = PrettyTable(
["User", "Start Time", "Status", "Backup File", "Conflict Info"]
)
with open(f'{path}/status.csv') as csvfile:
csvinfo = csv.DictReader(csvfile)
for row in csvinfo:
backupfile = None
conflict = None
if row['user'] in backup_file:
backupfile = backup_file[row['user']]
if os.path.isfile('/var/cpanel/users/{}'.format(row['user'])):
with open(
'/var/cpanel/users/{}'.format(row['user'])
) as userfile:
for line in userfile:
if line.startswith('OWNER='):
owner = line.split('=')[1].strip()
conflict = f'Account exists. Owner: {owner}'
row['status'] = '{}_with_Conflict'.format(row['status'])
x.add_row(
[
row['user'],
arrow.get(row['starttime']).humanize(),
row['status'],
backupfile,
conflict,
]
)
print(x)
print(f'There are currently {pending} pending transfers')
print(f'There are currently {transfer} active transfers')
return pending, transfer
def input_parse():
parser = argparse.ArgumentParser()
parser.add_argument(
'-e',
'--email',
action='store',
dest='Email',
help='Set email to send notification to',
)
parser.add_argument(
'-u', '--user', action='store', dest='User', help='Define your User'
)
parser.add_argument(
'-p',
'--pass',
action='store',
dest='ResPass',
default=None,
help='Set the password for the remote reseller. Not recommended to pass as an argument, but an option nonetheless.',
)
parser.add_argument(
'-r',
'--host',
action='store',
dest='RemoteHost',
default=None,
help='Remote Hostname.',
) # not -h because it conflicts with help
parser.add_argument(
'--ftphost',
action='store',
dest='FTPhost',
default=False,
help='Set hostname to use for FTP transfer',
)
parser.add_argument(
'--ftpuser',
action='store',
dest='FTPuser',
default=False,
help='Set username to use for FTP transfer',
)
parser.add_argument(
'--ftppass',
action='store',
dest='FTPpass',
default=False,
help='Set password for FTP transfer',
)
parser.add_argument(
'-s',
'--ssl',
action='store_true',
default=False,
dest='use_ssl',
help='Will make it run over port 2087 instead of 2086. Can cause SSL errors',
)
parser.add_argument(
'-n',
'--nosize',
action='store_true',
default=False,
dest='nosize',
help='Will make it run over port 2087 instead of 2086. Can cause SSL errors',
)
parser.add_argument(
'--skip',
action='store',
default=False,
nargs='+',
dest='skip',
help='Skip specific accounts. Useful with --all',
)
parser.add_argument(
'--accounts',
action='store',
default=False,
nargs='+',
dest='accounts',
help='Set an inclusive list of accounts to move. Skips the menu',
)
parser.add_argument(
'--concurrent',
action='store',
default=2,
dest='concurrent',
type=int,
help='Set how many backups will run at once when you use the --all option',
)
parser.add_argument(
'--timeout',
'-t',
action='store',
default=1800,
dest='timeout',
type=int,
help='Set a timeout in seconds to set item from Pending to Timed Out and set an available concurrency slot. Default: 1800 sec (30 min)',
)
parser.add_argument(
'--all',
action='store_true',
default=False,
dest='all',
help='Dump all backups for accounts within size specification',
)
parser.add_argument(
'-c',
'--createftp',
action='store',
default=False,
dest='createftp',
help='Create an FTP account automatically to do the transfer. Required for --status',
)
parser.add_argument(
'--status',
action='store',
default=False,
dest='status',
help='Provide a user or a path to check the status for',
)
parser.add_argument(
'--success',
action='store',
default=False,
dest='success',
help='Provide a path and it will show you all successful complete backups in that Directory',
)
parser.add_argument('--version', action='version', version='%(prog)s 1.0')
raw = parser.parse_known_args()
parsed = raw[0]
if parsed.status:
status_from_path(parsed.status, parsed.timeout)
sys.exit()
if parsed.success:
success_from_path(parsed.success)
sys.exit()
if not parsed.createftp:
print(
'It is strongly suggested you allow remote_dump to create the ftp user for you so it can track progress.'
)
create_ftp = yes_or_no('Would you like to create an ftp user?')
if create_ftp:
parsed.createftp = input(
"What cPanel account should the FTP user be created on: "
)
if parsed.createftp:
(
parsed.FTPhost,
parsed.FTPuser,
parsed.FTPpass,
parsed.ftproot,
) = create_ftp_user(parsed.createftp)
try:
maninput = False
if not parsed.User:
parsed.User = input(
"Enter the reseller user from the remote host: "
)
maninput = True
if not parsed.ResPass:
parsed.ResPass = input(
"Enter the reseller password for the remote host: "
)
maninput = True
if not parsed.RemoteHost:
parsed.RemoteHost = input("Enter remote hostname or IP address: ")
maninput = True
if not parsed.FTPhost:
parsed.FTPhost = input(
"Enter hostname for FTP transfer (new server): "
)
maninput = True
if not parsed.FTPuser:
parsed.FTPuser = input("Enter FTP user for new server: ")
maninput = True
if not parsed.FTPpass:
parsed.FTPpass = input(
'Enter Password for ' + parsed.FTPuser + ': '
)
maninput = True
welcome(parsed, maninput)
except KeyboardInterrupt as exc:
print('\nKeyboard Quit Detected')
remote_ftp_user(parsed, destroy=1)
sys.exit()
return parsed
def remote_ftp_user(parsed, destroy=0):
print('Removing FTP Account')
result = cpanel_api(
function='delftp',
module='Ftp',
version=2,
destroy=destroy,
user=parsed.FTPuser,
cpanel_jsonapi_user=parsed.createftp,
quota=0,
)
try:
if result['cpanelresult']['event']['result'] == 1:
print(f'Successfully removed FTP account {parsed.FTPuser}')
else:
print(f'Unable to remove FTP account {parsed.FTPuser}')
except KeyError as exc:
sys.exit(f'Unable To remove FTP account {parsed.FTPuser}')
def create_ftp_user(user):
if is_cpanel_user(user):
with open(f'/var/cpanel/users/{user}') as f:
for line in f:
if line.startswith('DNS='):
primarydomain = line.split('=')[1].strip()
break
else:
sys.exit('Not a valid user')
ftp_modifier = ''.join(
random.choice(
string.ascii_uppercase + string.digits + string.ascii_lowercase
)
for _ in range(6)
)
ftp_acct = f'imhxfer-{ftp_modifier}'
ftp_pass = ''.join(
random.choice(
string.ascii_uppercase + string.digits + string.ascii_lowercase
)
for _ in range(10)
)
hostname = socket.gethostname()
IP = socket.gethostbyname(hostname)
result = cpanel_api(
function='addftp',
module='Ftp',
version=2,
user=ftp_acct,
cpanel_jsonapi_user=user,
quota=0,
_data={'pass': ftp_pass},
)
if result['cpanelresult']['event']['result'] == 1:
print(
f'{IP} Created account {ftp_acct}@{primarydomain} with the password {ftp_pass}'
)
else:
sys.exit(
f'Unable to create FTP account. Output from cpanel API - {result}'
)
if os.path.isdir(f'/home/{user}/{ftp_acct}@{primarydomain}'):
ftp_doc_root = f'/home/{user}/{ftp_acct}@{primarydomain}'
elif os.path.isdir(f'/home/{user}/{ftp_acct}'):
ftp_doc_root = f'/home/{user}/{ftp_acct}'
else:
sys.exit('Unable to find Document Root of FTP account')
print(f'FTP Document root is {ftp_doc_root}')
return IP, f'{ftp_acct}@{primarydomain}', ftp_pass, ftp_doc_root
def remote_whm_post(parsed, jsoncall, **kwargs):
user = kwargs.get('user', parsed.User)
port = kwargs.get('port', '2086')
remoteHost = parsed.RemoteHost
if parsed.use_ssl:
port = 2087
url = 'https://' + remoteHost + ':' + port + '/' + jsoncall
else:
url = 'http://' + remoteHost + ':' + port + '/' + jsoncall
passholder = parsed.ResPass
encoded_pass = base64.b64encode(user + ':' + passholder)
auth = "Basic " + encoded_pass
header = {'Authorization': auth}
response = requests.post(url, headers=header, verify=False).json()
return response
def remote_cpanel_api(
host,
user,
cppass,
function,
version,
module,
ssl=False,
pos=None,
timeout=180,
_data=None,
**kwargs,
):
"""Make a POST request to the cPanel JSON API.
Args:
function (str): function name posted as cpanel_jsonapi_func
version (int): version of the API to use (1 or 2)
module (str): module name posted as cpanel_jsonapi_module
pos (list): list of positional args sent into the function as arg-0, arg-1, et al.
timeout (int, default 180): seconds to give the API to respond
All other data which must be supplied to the API can be
supplied as additional kwargs.
Special arg:
_data (dict): This shouldn't be necessary to use. If an
argument is impossible to send to the API as a kwarg for
whatever reason, it can be forced in using this. For example,
in python a kwarg cannot have a period in its name or begin
with a number. If you needed to supply stupid.arg as an argument,
you can do _data={'stupid.arg': value_here}"""
data = {} if _data is None else _data
# required kwargs. All others are supplied inside data{}
data['cpanel_jsonapi_module'] = module
data['cpanel_jsonapi_func'] = function
data['cpanel_jsonapi_apiversion'] = version
for key, val in kwargs.items():
data[key] = val
# positional args are sent in as post variables, in format
# &arg-0=SOMETHING&arg-1=SOMETHING
# the pos (pun intended) variable holds them in the order
# they are sent into the function
if isinstance(pos, str):
# even if there is only one arg, it should be a list
raise ValueError('pos should be a list, received string')
if isinstance(pos, list):
for pos_index, pos_arg in enumerate(pos):
data['arg-%d' % pos_index] = pos_arg
try:
print('%(yellow)sConnecting to remote cPanel API%(none)s' % col)
except OSError:
return None
try:
passholder = cppass
encoded_pass = base64.b64encode(user + ':' + passholder)
auth = "Basic " + encoded_pass
if ssl == True:
url = 'https://' + host + ':2087/json-api/cpanel'
else:
url = 'http://' + host + ':2086/json-api/cpanel'
return requests.post(
url,
data=data,
headers={'Authorization': auth},
timeout=timeout,
verify=False,
).json()
except (
requests.exceptions.RequestException,
ValueError, # JSON issues
TypeError, # JSON issues
):
return None
def biguser(response, parsed):
userlist = []
toobig = []
if 'cpanelresult' in response:
if 'error' in response['cpanelresult']:
print('%(red)sPossible Bad Password%(none)s' % col)
print('cPanel API Response:')
print(response['cpanelresult']['error'])
for i in range(len(response['data']['acct'])):
size = response['data']['acct'][i]['diskused']
if size == 'none':
userlist.append(
{
'user': response['data']['acct'][i]['user'],
'disk': response['data']['acct'][i]['diskused'].split('M')[
0
],
'quota': response['data']['acct'][i]['disklimit'],
}
)
if size != None and size != 'none':
size = int(size.split('M')[0])
if not parsed.nosize:
if size <= 6000:
userlist.append(
{
'user': response['data']['acct'][i]['user'],
'disk': response['data']['acct'][i][
'diskused'
].split('M')[0],
'quota': response['data']['acct'][i]['disklimit'],
}
)
else:
print('%(red)sAccount is over 6GB:%(none)s' % col)
print(response['data']['acct'][i]['user'])
toobig.append(
{
'user': response['data']['acct'][i]['user'],
'disk': response['data']['acct'][i][
'diskused'
].split('M')[0],
'quota': response['data']['acct'][i]['disklimit'],
}
)
else:
userlist.append(
{
'user': response['data']['acct'][i]['user'],
'disk': response['data']['acct'][i]['diskused'].split(
'M'
)[0],
'quota': response['data']['acct'][i]['disklimit'],
}
)
if size == None:
userlist.append(
{
'user': response['data']['acct'][i]['user'],
'disk': response['data']['acct'][i]['diskused'].split('M')[
0
],
'quota': response['data']['acct'][i]['disklimit'],
}
)
return (userlist, toobig)
def gen_backup_select(userdict, parsed):
select_user_list = []
for i in range(len(userdict)):
select_user_list.append(userdict[i]['user'])
if parsed.all:
return select_user_list
if parsed.accounts:
return parsed.accounts
else:
opts = picker.Picker(
title='Select cPanel user to transfer',
options=sorted(select_user_list),
).get_selected()
if opts == False:
print("Aborted!")
else:
print(opts)
return opts
def backup_user(parsed, mvusers):
users_to_mv = gen_backup_select(mvusers, parsed)
if parsed.createftp:
if os.path.isdir(parsed.ftproot):
statusfile = f'{parsed.ftproot}/status.csv'
with open(statusfile, 'w') as status:
status.write('user,status,starttime\n')
for user in sorted(users_to_mv):
status.write(f'{user},Waiting,{time.time()}\n')
else:
sys.exit(f'Unable to find FTP directory {parsed.ftproot}')
dest = 'passiveftp'
server = parsed.FTPhost
user = parsed.FTPuser
ftppass = parsed.FTPpass
if not parsed.Email:
email = ('docs@inmotionhosting.com',)
else:
email = parsed.Email
port = ('21',)
rdir = ('bkup',)
for cpuser in sorted(users_to_mv):
if parsed.createftp:
while True:
pending, transferring = status_from_path(
f'{parsed.ftproot}', parsed.timeout
)
if pending < parsed.concurrent:
print('Starting Backup for ' + cpuser)
open(f'{parsed.ftproot}/{cpuser}.started', 'a').close()
result = remote_cpanel_api(
host=parsed.RemoteHost,
user=parsed.User,
cppass=parsed.ResPass,
version=1,
ssl=True,
module='Fileman',
function='fullbackup',
cpanel_jsonapi_user=cpuser,
pos=[dest, server, user, ftppass, email, port, rdir],
)
try:
# print result
if result['event']['result'] == 1:
# print '%(green)sSuccess%(none)s' % col
update_csv(
cpuser,
'Pending',
f'{parsed.ftproot}/status.csv',
)
break
except KeyError:
print(result)
break
else:
print(
f'Already {parsed.concurrent} transfers running. Pausing for 15 seconds then checking again'
)
time.sleep(15)
else:
print('Starting Backup for ' + cpuser)
result = remote_cpanel_api(
host=parsed.RemoteHost,
user=parsed.User,
cppass=parsed.ResPass,
version=1,
ssl=True,
module='Fileman',
function='fullbackup',
cpanel_jsonapi_user=cpuser,
pos=[dest, server, user, ftppass, email, port, rdir],
)
try:
# print result
if result['event']['result'] == 1:
print(
strcolor(
'green', f'Successfully started Backup for {cpuser}'
)
)
except KeyError:
print(result)
def test_ftp_creds(parsed):
print('Testing FTP Login')
server = parsed.FTPhost
user = parsed.FTPuser
password = parsed.FTPpass
try:
ftp = ftplib.FTP(server)
ftp.login(user, password)
except Exception as e:
print(e)
print(
"Unable to connect via FTP. The transfer will fail. Check your FTP credentials and submit again"
)
quit()
print('%(green)sFTP Connection Established%(none)s' % col)
def validate_cpanel_connection(parsed):
s = socket.socket()
address = parsed.RemoteHost
port = 2086 # port number is a number, not string
try:
s.connect((address, port))
except Exception as e:
print(
"Unable to connect to {} on port {}. Check to see if valid connection".format(
address, port
)
)
quit()
def print_notice(parsed):
if not parsed.Email:
print(
'%(yellow)sEmail response from remote API will go to docs@inmotionhosting.com instead of mht@inmotionhosting.com%(none)s'
% col
)
else:
print(
strcolor(
'yellow',
f'Email response from remote API will go to {parsed.Email}',
)
)
def main():
parsed = input_parse()
print_notice(parsed)
validate_cpanel_connection(parsed)
test_ftp_creds(parsed)
print('Determining the accounts to move by size')
userlists = biguser(
remote_whm_post(
parsed,
'json-api/listaccts?api.version=1&want=user,disklimit,diskused',
),
parsed,
)
mvusers = userlists[0]
if parsed.skip:
mvuser_copy = list(mvusers)
for entry in mvusers:
if str(entry['user']) in parsed.skip:
mvuser_copy.remove(entry)
mvusers = list(mvuser_copy)
bigusers = userlists[1]
print(
'%(green)sThe following users are the correct size to be backed up from the remote host%(none)s'
% col
)
x = PrettyTable(['User', 'Disk Space in MB'])
for entry in mvusers:
if parsed.accounts:
if entry['user'] in parsed.accounts:
x.add_row([entry['user'], entry['disk']])
else:
x.add_row([entry['user'], entry['disk']])
print(x)
input("Press Enter to continue...")
backup_user(parsed, mvusers)
if parsed.createftp:
pending, transferring = status_from_path(
f'{parsed.ftproot}/', parsed.timeout
)
while True:
if pending > 0 or transferring > 0:
print(pending, transferring)
pending, transferring = status_from_path(
f'{parsed.ftproot}', parsed.timeout
)
time.sleep(10)
else:
print(f'Pending: {pending} \nTransferring {transferring}')
break
if __name__ == "__main__":
main()
Zerion Mini Shell 1.0