Memory Optimization

Memory Profiling

import numpy as np
import tracemalloc

# Check size
arr = np.zeros((1000, 1000))
print(f"Memory: {arr.nbytes / 1024**2:.2f} MB")

# Track usage
tracemalloc.start()
result = my_function()
_, peak = tracemalloc.get_traced_memory()
print(f"Peak: {peak / 1024**2:.2f} MB")
tracemalloc.stop()

Data Types

# Choose smallest dtype
arr_float64 = np.zeros(1000000)              # 8 MB
arr_float32 = np.zeros(1000000, dtype=np.float32)  # 4 MB
arr_int16 = np.zeros(1000000, dtype=np.int16)      # 2 MB

In-Place Operations

# Bad: Creates temporaries
result = a + b + c

# Good: In-place
result = a.copy()
result += b
result += c

# Use out parameter
np.add(a, b, out=result)

Sparse vs Dense

from scipy import sparse

# Use sparse when density < 10%
n = 10000
dense_mem = n * n * 8 / 1024**2  # 762 MB
sparse_mem = (nnz * 12) / 1024**2  # ~11 MB at 1% density

Memory-Mapped Arrays

# For arrays larger than RAM
mmap = np.memmap('large.dat', dtype='float64',
                 mode='w+', shape=(100000, 1000))
mmap[:100] = np.random.randn(100, 1000)
del mmap  # Flush

# Read
mmap = np.memmap('large.dat', dtype='float64',
                 mode='r', shape=(100000, 1000))

Pre-allocation

# Bad: Growing list
results = []
for i in range(n):
    results.append(compute(i))
results = np.array(results)

# Good: Pre-allocate
results = np.empty(n)
for i in range(n):
    results[i] = compute(i)

Views vs Copies

# View (no allocation)
view = arr[::2]

# Copy (allocates)
copy = arr[::2].copy()
np.shares_memory(arr, view)  # True