[LibOS] Fix some bugs related to O_DIRECTORY handling

LibOS/shim/src/fs/shim_namei.c

@@ -457,11 +457,13 @@ int __path_lookupat (struct shim_dentry * start, const char * path, int flags,
             *dent = my_dent;
 
         // Enforce LOOKUP_CREATE flag at a higher level
-        if (my_dent->state & DENTRY_NEGATIVE)
-                err = -ENOENT;
+        if (my_dent->state & DENTRY_NEGATIVE) {
+            err = -ENOENT;
+            goto out;
+        }
 
         // Enforce the LOOKUP_DIRECTORY flag
-        if ((flags & LOOKUP_DIRECTORY) & !(my_dent->state & DENTRY_ISDIRECTORY))
+        if ((flags & LOOKUP_DIRECTORY) && !(my_dent->state & DENTRY_ISDIRECTORY))
             err = -ENOTDIR;
     }
 

LibOS/shim/src/sys/shim_fs.c

@@ -180,7 +180,7 @@ int shim_do_rmdir(const char* pathname) {
     dent->state |= DENTRY_NEGATIVE;
 out:
     put_dentry(dent);
-    return 0;
+    return ret;
 }
 
 mode_t shim_do_umask(mode_t mask) {