I was able to isolate the problem to fbi, I installed feh instead and now everything works perfect! And I figured out how to add more modes.
Here’s the final code and a schematic for those interested. Thanks everyone for the help and especially @MRAB 
from gpiozero import Button
from subprocess import Popen, PIPE
from time import sleep
def start_task(mode):
if mode == 1:
return Popen("exec python3 mode.py", cwd="/home/pi/project/mode1", shell=True, stdout=PIPE)
elif mode == 2:
return Popen("exec feh -Z -x -F -Y -B -q -D 5 *", cwd="/home/pi/project/mode2/images", shell=True)
elif mode == 3:
return Popen("exec cvlc --no-video-title-show -L -f mode3.mp4", cwd="/home/pi/project/mode3", shell=True, stdout=PIPE)
elif mode == 4:
return Popen("exec cvlc --no-video-title-show -L -f mode4.mp4", cwd="/home/pi/project/mode4", shell=True, stdout=PIPE)
else:
return None
previous_button = Button(2)
next_button = Button(4)
currently_pressed = None
current_mode = 1
current_task = start_task(current_mode)
while True:
if previous_button.is_pressed:
if currently_pressed is None:
currently_pressed = previous_button
if current_mode == 1:
current_mode = 4
else:
current_mode -= 1
current_task.terminate()
current_task = start_task(current_mode)
elif next_button.is_pressed:
if currently_pressed is None:
currently_pressed = next_button
if current_mode == 4:
current_mode = 1
else:
current_mode += 1
current_task.terminate()
current_task = start_task(current_mode)
else:
currently_pressed = None
sleep(0.25)
