AWS sagemaker image_uri processor: pip install fails #71
Description
Hi, thanks for a useful package.
Can you provide some guidance on installing this in a sagemaker enviroment. I get exit errors after trying to pip install flatterer with a few sagemaker images, all with python 3.7 at least. I also tried installing rust per documentation but to no avail. Here is an example I tried tried:
_**
from sagemaker.processing import ScriptProcessor
processor = ScriptProcessor(
role='arn:aws:iam::xxxxxxxxxxx:role/AWS-Sagemaker',
image_uri='xxxxxxxxxxx.dkr.ecr.<region>.amazonaws.com/sagemaker-scikit-learn:0.20.0-cpu-py3',
command=['python3'],
instance_count=1,
instance_type='ml.r5.2xlarge',
base_job_name='test_report'
)
processor.run(
code='runreport.py',
source_dir='source_dir'
)
```**_
in my runreport.py I have this code
**_from time import time
import subprocess
import warnings
warnings.filterwarnings('ignore')
def install_packages(packages):
print('Installing Packages...!')
i = 1
for p in packages:
try:
# Check if the package is already installed
subprocess.check_output([sys.executable, '-m', 'pip', 'show', p])
print(str(i)+'/'+str(len(packages))+ ': '+ p + ' is already installed')
except subprocess.CalledProcessError:
# Package is not installed, install it
try:
subprocess.check_call([sys.executable, '-m', 'pip', 'install', p], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(str(i)+'/'+str(len(packages))+ ': Installed '+p)
except subprocess.CalledProcessError as e:
print(f"Error installing {p}: {e.output}")
i = i+1
print('Done!')
print('----------------------------------------')
def install_rust_toolchain():
try:
print('Installing Rust toolchain...')
subprocess.check_call(['curl', '--proto', '=https', '--tlsv1.2', '-sSf', 'https://sh.rustup.rs', '-o', 'rustup-init.sh'])
subprocess.check_call(['sh', 'rustup-init.sh', '-y'])
os.environ['PATH'] += os.pathsep + os.path.expanduser('~/.cargo/bin')
print('Rust toolchain installed successfully.')
except subprocess.CalledProcessError as e:
print(f"Failed to install Rust toolchain: {e}")
sys.exit(1)
if name == 'main':
try:
script_dir = os.path.dirname(os.path.realpath(file))
print(script_dir)
install_rust_toolchain()
install_packages(packages=['sagemaker', 'flatterer', ...])
_**
All other packages installs ok except for flatterer.