My below main.py code gives MemoryError: memory allocation failed, allocating 72 bytes
I'm using ESP8266 12E. I understand the heap is fragmented, but can someone give an idea of where in this code would the memory issue be happening at? The code first runs a web server where from a user can configure the home SSID and password. Once that is set, next time on, it opens a socket connection with someserver.com.
Please if someone could point me to how to fix this, I'd be grateful.
import gc
import network
gc.collect()
import machine
gc.collect()
import ubinascii
gc.collect()
import ujson
gc.collect()
import uos
gc.collect()
import utime
gc.collect()
import socket
gc.collect()
import uselect
gc.collect()
import urequests
gc.collect()
from uhashlib import sha256
gc.collect()
import ustruct
gc.collect()
html = """<!DOCTYPE html>
<html>
<head> <title>Ouroboros IoT Login</title> </head>
<body>
<form action="http://192.168.0.1/configure.html" method="post">
Username : <input type="text" name="username"><br>
Password: <input type="password" name="password" ><br>
<input type="submit" value="submit" name="submit">
</form>
</body>
</html>
"""
login_fail_html = """<!DOCTYPE html>
<html>
<head> <title>Ouroboros IoT Login</title> </head>
<body>
<h2>Incorrect Credentials!</h2><br>Please login<br>
<form action="configure.html" method="post">
Username : <input type="text" name="username"><br>
Password: <input type="password" name="password" ><br>
<input type="submit" value="submit" name="submit">
</form>
</body>
</html>
"""
# Last Time Stamp when KeepAlive was sent
last_keepalive_sent = None
# Is device connected to server
iot_device_connected = False
# This contains the summary of the device status, whether the device is ON/OFF, and if there is a delayed ON/OFF policy
device_status_map = {'device_0':{'curr_status':0, 'on_time':None, 'off_time': None}, 'device_2':{'curr_status':0, 'on_time':None, 'off_time': None}}
# Updates the status of the device
def update_device_status(device_num, device_status, on_time = None, off_time = None):
device_num_key = 'device_' + str(device_num)
# Check if the key exists in device_status MAP
if device_num_key in device_status_map:
device_status_obj = device_status_map[device_num_key]
# Update the device's status value in map
device_status_obj['curr_status'] = device_status
device_status_obj['on_time'] = on_time
device_status_obj['off_time'] = off_time
# Now turn ON/OFF the device
if device_status == 1:
turn_gpio_on(device_num)
elif device_status == 0:
turn_gpio_off(device_num)
# Perform tasks that should be done periodically
def perform_cron_task(sock):
# Get current time since epoch
curr_time = utime.time()
# check if its time to turn ON/OFF a device
for device_num_key in device_status_map:
# Parse device Number
device_num = int(device_num_key[len('device_')])
# Get status
device_status_obj = device_status_map[device_num_key]
# Get the current status and the off_time
on_time = device_status_obj['on_time']
off_time = device_status_obj['off_time']
status = device_status_obj['curr_status']
# If the device is OFF and has on_time set:
print("Just before")
if (on_time is not None) and (status == 0) and (curr_time >= on_time):
print("A")
update_device_status(device_num, 1, on_time = None, off_time = None)
elif (off_time is not None) and (status == 1) and (curr_time >= off_time):
print("A")
update_device_status(device_num, 0, on_time = None, off_time = None)
print("B last_keepalive_sent : " + str(last_keepalive_sent) + ", curr_time : " + str(curr_time))
if (last_keepalive_sent is not None) and ((curr_time - last_keepalive_sent) > 60):
# Send the status packet every 60 seconds
# TODO : This is wrong!! would be sending status packets every 1 sec!!
print("C")
send_status_packet(sock)
# Sends the Status Packet to server
def send_status_packet(sock):
version = 1
pkt_type = 10
# Size of the dev status map is total number of devices
num_dev = len(device_status_map)
# Version(1 byte) + Pkt Type (2 bytes) Num Devices (1 bytes) + Device A offset (1 byte) + Device A Status (1 byte) + Device A Status Extra Info (4 bytes) + ...
# dataToSend = version.to_bytes(1, byteorder='big') + pkt_type.to_bytes(2, byteorder='big') + num_dev.to_bytes(1, byteorder='big')
dataToSend = ustruct.pack('>BHB', version, pkt_type, num_dev) # version.to_bytes(1, byteorder='big') + pkt_type.to_bytes(2, byteorder='big') + num_dev.to_bytes(1, byteorder='big')
# Now iterate over the devices status map
for device_num_key in device_status_map:
# Parse device Number
device_num = int(device_num_key[len('device_')])
# Get status
device_status_obj = device_status_map[device_num_key]
device_status = device_status_obj['curr_status']
# Add the bytes for this device into dataToSend
# TODO Later : Fill something in 4 bytes of extra information
extra_info = 0
dataToSend = dataToSend + ustruct.pack('>BBL', device_num, device_status, extra_info) # device_num.to_bytes(1, byteorder='big') + device_status.to_bytes(1, byteorder='big') + extra_info.to_bytes(4, byteorder='big')
# Send the packet now
print("Sending Status Packet")
print(dataToSend)
sock.send(dataToSend)
global last_keepalive_sent
last_keepalive_sent = utime.time()
# Updates the Real Time Clock with the provided Epoch
def update_rtc(epoch):
cal_datetime = utime.localtime(epoch)
machine.RTC().datetime(cal_datetime)
# Make GET REST API call to url
def http_get(url):
return urequests.get(url)
# Check if file exists
def fileExists(fileName):
try:
uos.stat(fileName)
print("File " + fileName + " found!")
return True
except OSError:
print("No file " + fileName + " found!")
return False
# Turns WiFi ON for configuration
def turn_wifi_on():
# Setup the AP interface
ap_if = network.WLAN(network.AP_IF)
ap_if.active(False)
utime.sleep_ms(500)
utime.sleep_ms(500)
utime.sleep_ms(500)
utime.sleep_ms(500)
ap_if.active(True)
utime.sleep_ms(500)
utime.sleep_ms(500)
utime.sleep_ms(500)
utime.sleep_ms(500)
# Get the MACADDRESS - without any spaces
macaddress = ubinascii.hexlify(ap_if.config('mac'),':').decode()
macaddress = macaddress.replace(':','')
ap_if.config(essid="OUB1_"+macaddress, password="12345678")
#ap_if.config(essid="OUB1_"+macaddress)
ap_if.ifconfig(('192.168.0.1', '255.255.255.0', '192.168.0.1', '192.168.0.1'))
# Configure the AP to static IPs
def turn_wifi_off():
ap_if = network.WLAN(network.AP_IF)
ap_if.active(False)
utime.sleep_ms(500)
utime.sleep_ms(500)
utime.sleep_ms(500)
utime.sleep_ms(500)
# Find out the stored IoT secret content
def get_iot_secret():
fileName = 'alpha.txt'
if fileExists(fileName):
f = open(fileName)
content_str = f.read()
f.close()
return content_str
else:
return 'asdasrefwefefergf9rerf3n4r23irn1n32f'
# Find out the stored home network credential if exist
def get_wifi_config():
fileName = 'wifi.conf'
if fileExists(fileName):
f = open(fileName)
content_str = f.read()
f.close()
content = ujson.loads(content_str)
return content
else:
return None
# Set the home network credentials
def save_wifi_config(essid, passphrase):
f = open('wifi.conf', 'w')
config = {'essid':essid, 'passphrase':passphrase}
config_str = ujson.dumps(config)
f.write(config_str)
f.close()
# Find out the stored login credentials
def get_login_config():
fileName = 'login.conf'
if fileExists(fileName):
f = open(fileName)
content_str = f.read()
f.close()
content = ujson.loads(content_str)
return content
else:
# No file exists so far, so use the admin/admin credentials
return {'user':'admin','password':'admin'}
# Set the login credentials
def save_login_config(user, password):
f = open('login.conf', 'w')
config = {'user':user, 'password':password}
config_str = ujson.dumps(config)
f.write(config_str)
f.close()
def turn_gpio_on(device_num):
# Device Num to Pin Mapping
if device_num == 0:
pin_num = 0
elif device_num == 1:
pin_num = 2
# Check Pin
pin = machine.Pin(pin_num)
if pin.value() == 0:
pin.on()
# else it is already at HIGH state, nothing to do
def turn_gpio_off(device_num):
# Device Num to Pin Mapping
if device_num == 0:
pin_num = 0
elif device_num == 1:
pin_num = 2
# Check Pin
pin = machine.Pin(pin_num)
if pin.value() == 1:
pin.off()
# else it is already at LOW state, nothing to do
def init_pin(device_num):
# Device Num to Pin Mapping
if device_num == 0:
pin_num = 0
elif device_num == 1:
pin_num = 2
#open GPIO0 in output mode & turn it off by default
pin = machine.Pin(pin_num, machine.Pin.OUT)
# Turn off both GPIO initially
turn_gpio_off(device_num)
def send_keep_alive(sock, epoch, md5hash):
# version (1 byte) + Packet Type(2 bytes) + MAC (6 bytes) + TimeStamp (4 bytes) + SHA256-Hash of Secret.TimeStamp (32 bytes)
cl_if = network.WLAN(network.STA_IF)
print("SendKeepAlive A")
# Get the MACADDRESS - without any spaces
macaddress = ubinascii.hexlify(cl_if.config('mac'),':').decode()
macaddress = macaddress.replace(':','')
# version (1 byte) + Packet Type(2 bytes) + MAC (6 bytes) + TimeStamp (4 bytes) + SHA256-Hash of Secret.TimeStamp (32 bytes)
version = 1
pkt_type = 1
print("SendKeepAlive B")
# version.to_bytes(1, byteorder='big') + pkt_type.to_bytes(2, byteorder='big') + ubinascii.unhexlify(macaddress) + epoch.to_bytes(4, byteorder='big') + md5hash
dataToSend = ustruct.pack('>BH', version, pkt_type) + ubinascii.unhexlify(macaddress) + ustruct.pack('>L', epoch) + md5hash
# Send the packet now
print("SendKeepAlive C")
sock.send(dataToSend)
print("SendKeepAlive D")
def process_keepalive_ack(sock, payload):
print("Got Keepalive ACK from server!")
status_pkt = payload[0:1]
# status = int.from_bytes(status_pkt, byteorder='big')
status = ustruct.unpack('>B', status_pkt)[0]
print("status : " + str(status))
if status == 1:
print("Got ACK Status 1")
global iot_device_connected
iot_device_connected = True
print("Sending the Status Packet to Server! last_keepalive_sent : " + str(last_keepalive_sent))
send_status_packet(sock)
def process_command_pkt(payload):
# Packet Length check should be done
if len(payload) >= 4:
cmd_subtype_pkt = payload[0:2]
# cmd_sub_type = int.from_bytes(cmd_subtype_pkt, byteorder='big')
cmd_sub_type = ustruct.unpack('>H', cmd_subtype_pkt)[0]
# Depending on sub command, the payload would differ
if cmd_sub_type == 1:
# Indefinite ON
dev_index_pkt = payload[2:4]
# dev_index = int.from_bytes(dev_index_pkt, byteorder='big')
dev_index = ustruct.unpack('>H', dev_index_pkt)[0]
update_device_status(dev_index, 1, on_time = None, off_time = None)
elif cmd_sub_type == 2:
# Indefinite OFF
dev_index_pkt = payload[2:4]
# dev_index = int.from_bytes(dev_index_pkt, byteorder='big')
dev_index = ustruct.unpack('>H', dev_index_pkt)[0]
# Turn OFF
update_device_status(dev_index, 0, on_time = None, off_time = None)
elif (cmd_sub_type == 3) and len(payload) >= 6:
dev_index_pkt = payload[2:4]
# dev_index = int.from_bytes(dev_index_pkt, byteorder='big')
dev_index = ustruct.unpack('>H', dev_index_pkt)[0]
# Get the time period for which this has to be ON
sec_time_on_pkt = payload[4:6]
# sec_time_on = int.from_bytes(sec_time_on_pkt, byteorder='big')
sec_time_on = ustruct.unpack('>H', sec_time_on_pkt)[0]
print("CMD Type 3 : dev_index : " + str(dev_index) + ", sec_time_on:" + str(sec_time_on))
# Compute the Off Time:
off_time = utime.time() + sec_time_on
# Turn ON with a timer
update_device_status(dev_index, 1, on_time = None, off_time = off_time)
# Process packet received from the server
def process_server_packet(data, sock):
# The packet should have more than 6 bytes
if len(data) > 6:
pkt_type_pkt = data[0:2]
epoch_pkt = data[2:6]
# pkt_type = int.from_bytes(pkt_type_pkt, byteorder='big')
pkt_type = ustruct.unpack('>H', pkt_type_pkt)[0]
# epoch = int.from_bytes(epoch_pkt, byteorder='big')
epoch = ustruct.unpack('>L', epoch_pkt)[0]
# Check if pkt_type is valid:
if (pkt_type == 2) or (pkt_type == 3):
# Set the RTC
update_rtc(epoch)
# Packet Handler
if pkt_type == 2:
# This is ack for keepalive
process_keepalive_ack(sock, data[6:])
elif pkt_type == 3:
# This is a command packet
process_command_pkt(data[6:])
# Now send the current status packet
send_status_packet(sock)
def recv_timeout(the_socket,timeout=2):
#make socket non blocking
the_socket.setblocking(0)
#total data partwise in an array
toret_data='';
#beginning time
begin=utime.time()
print("timeout : " + str(timeout))
print("Begin : " + str(begin))
while 1:
now = utime.time()
print("now : " + str(now))
#if you got some data, then break after timeout
if (len(toret_data)>0) and (now - begin > timeout):
break
#if you got no data at all, wait a little longer, twice the timeout
elif (now - begin) > (timeout*2):
break
#recv something
try:
print("Trying to receive Before")
data = the_socket.recv(100)
data = data.decode()
print("Trying to receive after")
if data:
toret_data = toret_data + data
#change the beginning time for measurement
begin=utime.time()
print("got some bytes : " + str(len(data)))
print("Resetting begin time to " + str(begin))
else:
#sleep for sometime to indicate a gap
print("No data found in recv, sleeping for 200 ms")
utime.sleep_ms(200)
except Exception as e:
print("Got some exception\r\n")
print(str(e))
utime.sleep_ms(200)
pass
#join all parts to make final string
return toret_data
# Find out the post parameters in a dictionary
def get_post_params(req):
print("Inside GET POST PARAMS : req = " + req)
post_params = req.split('\r\n')[-1:][0]
# Check if the post body contains the necessary fields
# Split the post_params by &
# params : ['username=', 'password=', 'method=POST', 'url=http%3A%2F%2Ftwig-me.com%2Fv1%2Fusergroups%2FWKMUYXELA9LCC', 'jsondata=', 'submit=submit']
print("post_params : " + post_params)
params = post_params.split('&')
print("Params")
print(params)
# Initialize the key value pair dict
post_dict = {}
# Iterate on each param
for param in params:
# Each param would be like 'method=POST', etc
key_val = param.split('=')
print("Key Val :")
print(key_val)
key = key_val[0]
val = key_val[1]
# Update post_dict
post_dict[key] = val
return post_dict
# Connect to the server by opening a Socket
def connect_server():
r = http_get('http://someserver.com/v13/gettime')
to_ret = {'code': r.status_code, 'reply': r.text}
# JSON Loads
output = ujson.loads(to_ret['reply'])
# Check error, and data
print("Output " + r.text)
if output['error'] == False:
# The returned value in data is the Time Since Epoch on server
epoch = output['data']
try:
# Now try to connect to the server
print("Before socket.gteaddrinfo")
sockaddr = socket.getaddrinfo('manukahoneyindia.com', 80)[0][-1][0]
print(sockaddr)
# Create socket
sock = socket.socket()
sock.connect((sockaddr, 8000))
# Set to non blocking
sock.setblocking(0)
# Get secret to generate the hash
secret = get_iot_secret()
# Find the hash of secret+ts
hash_bytes = sha256(secret + str(epoch)).digest()
# Send the keepalive packet now
send_keep_alive(sock, epoch, hash_bytes)
poller = uselect.poll()
poller.register(sock, uselect.POLLERR | uselect.POLLIN | uselect.POLLHUP)
# Poll for a sec
loop_break = 0
print("gamma")
while True and (loop_break == 0):
events = poller.poll(500) # time in milliseconds
print(events)
if events:
for fd, flag in events:
print(flag)
if flag & uselect.POLLIN:
# Event is there, means data is there, read the data
# 250 bytes should be more than enough since most packets
# would be not more than 100 bytes
data = sock.recv(250)
# Now try to do processing of the data received
if len(data) == 0:
# Most likely the socket has closed on server!
poller.unregister(sock)
loop_break = 1
break
else:
process_server_packet(data, sock)
if (flag & uselect.POLLHUP) or (flag & uselect.POLLERR):
poller.unregister(sock)
loop_break = 1
break
# Do periodic Tasks
perform_cron_task(sock)
# Sleep for a small while - 500 ms
utime.sleep_ms(500)
utime.sleep_ms(500)
utime.sleep_ms(500)
except OSError:
if 'poller' in locals():
poller.unregister(sock)
finally:
sock.close()
# Now set the status of the device to Not Connected
global iot_device_connected
iot_device_connected = False
# Try to connect the STATION
def do_connect():
# Use my personal wifi credentials which are stored in a file
wifi_config = get_wifi_config()
# Check if home wifi config is valid, if so, connect to it
if wifi_config is not None:
# 'JioFi2_0BC73C', '6uys4rdixs'
home_wifi_ssid = wifi_config['essid']
home_wifi_pwd = wifi_config['passphrase']
# Activate the station interface
sta_if = network.WLAN(network.STA_IF)
sta_if.active(False)
utime.sleep_ms(500)
utime.sleep_ms(500)
utime.sleep_ms(500)
sta_if.active(True)
# Connect to the home WiFi network
sta_if.connect(home_wifi_ssid, home_wifi_pwd)
# Keep checking connectivity, and query/ping server
while True:
# Keep on waiting until the station is connected
while not sta_if.isconnected():
machine.idle() # save power while waiting
# Connected!
print('network config:', sta_if.ifconfig())
# Came here means the Station is connected!
try:
connect_server()
except OSError:
print("Got OS Error! Maybe the internet connection or API failed?")
# Sleep for a while
utime.sleep_ms(500)
else:
# Wifi config is not set
# Should not have come here in that case!
# Open the web server indefinitely
start_web_server()
# This web server takes care of the WiFi configuration
# max_run_sec
def web_server(max_run_sec = None):
# Turn wifi interface ON
turn_wifi_on()
# Create server socket
addr = socket.getaddrinfo('0.0.0.0', 80)[0][-1]
s = socket.socket()
# TODO : If both the wifi and sta are operating simultaneously, then bind only to WiFi
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(addr)
s.listen(1)
# s.settimeout(1)
wdt = machine.WDT()
print("Just before registering poller")
poller = uselect.poll()
poller.register(s, uselect.POLLIN)
# Get the current time since epoch
startTimeEpoch = utime.time()
while True:
if max_run_sec is not None:
elapsedTime = utime.time() - startTimeEpoch
if elapsedTime > max_run_sec:
# Max run time of web server has elapsed, time to exit this mode!
break
#print("Just before polling")
all_events = poller.poll(400) # time in milliseconds
#all_events = poller.ipoll(200)
#print("Just after polling!")
print(all_events)
read_avl = 0
#for (obj, all_event) in all_events:
#for f, ev, data in all_events:
# read_avl = 1
#for key, val in all_events:
# print(key)
# print(val)
# read_avl = 1
#print("Just after FOR Loop")
if len(all_events) > 0:
#if read_avl == 1:
print("Just before TRY!")
try:
print("Just after GC Collect!")
gc.collect()
#print("Just before accepting")
res = s.accept()
client_s = res[0]
client_addr = res[1]
req = ''
#while True:
# data = client_s.recv(200)
# if data:
# req += str(data, 'utf8')
# else:
# break
# utime.sleep_ms(50)
# Original
#req = client_s.recv(4096)
# TODO : Also try this :
req = recv_timeout(client_s)
#req = req.decode()
print(req)
req = str(req)
# Came here means that there has been some connection!
# Reset the start time epoch in such a case:
startTimeEpoch = utime.time()
# Check route now
if req.find('conf_wifi.html') != -1:
# Check if the username and password are correct, if not, configure:
print("Inside conf_wifi.html")
login_config = get_login_config()
username = login_config['user']
pwd = login_config['password']
# Get post parameters
print("Just before get post params")
post_dict = get_post_params(req)
print("Just after get post params")
# Now check if the post_dict has the key and value for username and password as needed?
username_post = post_dict['username']
password_post = post_dict['password']
# Check if the password is same as expected
if (username_post == username) and (password_post == pwd):
print("Password and UserName match")
# Do some sanity check for handling the new wifi ssid and password
new_wifi_ssid = post_dict['essid']
new_wifi_passphrase = post_dict['passphrase']
# Set the wifi credentials
save_wifi_config(new_wifi_ssid, new_wifi_passphrase)
client_s.send('<!DOCTYPE html><html><head> <title>Ouroboros IoT WiFi Configuration Success</title> </head><body>Configuration successful!<br>Device would go into reboot now!</body></html>')
client_s.close()
# Sleep for some time so that the page could get rendered
utime.sleep(4)
# Reboot device now
machine.reset()
else:
client_s.send(login_fail_html)
elif req.find('configure.html') != -1:
print("Got configure request!\r\n")
# Check if the username and password are correct, if not, configure:
login_config = get_login_config()
username = login_config['user']
pwd = login_config['password']
print("Username : " + username + ", pwd : " + pwd)
# Find the POST PARAMETERS sent
# There would be just one entry in the array, so get the 0th index directly
# post_params : 'username=&password=&method=POST&url=http%3A%2F%2Ftwig-me.com%2Fv1%2Fusergroups%2FWKMUYXELA9LCC&jsondata=&submit=submit'
print("Came here A")
post_dict = get_post_params(req)
# Now check if the post_dict has the key and value for username and password as needed?
username_post = post_dict['username']
password_post = post_dict['password']
print("Came here B")
# Check if the password is same as expected
if (username_post == username) and (password_post == pwd):
hidden_input = '<input type="hidden" name="username" value="' + username + '"><input type="hidden" name="password" value="' + pwd + '">'
# Send the login username and password inside the hidden input field
configure_html = '<!DOCTYPE html><html><head> <title>Ouroboros IoT WiFi Configuration Page</title> </head><body><form action="http://192.168.0.1/conf_wifi.html" method="post">WiFi SSID : <input type="text" name="essid"><br>WiFi Password: <input type="password" name="passphrase" ><br>' + hidden_input + '<input type="submit" value="submit" name="submit"></form></body></html>'
# TODO : Also show link to webpage, where from we can change the login credentials
client_s.send(configure_html)
else:
client_s.send(login_fail_html)
elif req.find('index.html') != -1:
print("Got index.html request!\r\n")
client_s.send(html)
else :
# Do nothing
print("Invalid request received! Show the login page again!\r\n")
client_s.send(html)
except OSError:
# Got no request and it timedout!
print("Timed-out, no request received!\r\n")
except Exception as e:
print("Got some exception\r\n")
print(str(e))
finally:
client_s.close()
machine.idle()
utime.sleep_ms(50)
machine.idle()
wdt.feed()
# Unregister poller
poller.unregister(s)
# When while loop ends!
s.close()
# Turn wifi interface OFF
turn_wifi_off()
# Starts a thread which runs the web server to handle WiFi
def start_web_server(max_run_sec = None):
# start_new_thread(web_server, (max_run_sec))
web_server(max_run_sec)
############# MAIN ##########################
# Initialize two pins to INPUT and OFF by default
init_pin(0)
init_pin(1)
# Check if the home wifi network has been setup
# Check if home wifi config is valid, if so, connect to it
# If home wifi is not configured, then use the Web server all the time.
try:
if get_wifi_config() is None:
# Came here means the wifi is not configured
# Start the web server
print("Starting web server")
start_web_server()
else:
# In such case, we should still run the web server for first 120 seconds!
start_web_server(30)
# Now the station interface
do_connect()
finally:
# If somehow came here means machine should go for reset?
init_pin(0)
init_pin(1)
machine.reset()