diff ignore files only in one directory

diff -bBur old_dir/ new_dir/ | grep -v "^Only in" > my.diff